diff --git a/performa/engine/report.py b/performa/engine/report.py index 0f0769f..4404311 100644 --- a/performa/engine/report.py +++ b/performa/engine/report.py @@ -117,6 +117,8 @@ def generate_report(scenario, base_dir, mongo_url, db_name, doc_folder, report_definition = scenario['report'] report_template = report_definition['template'] + LOG.info('Using report template: %s', report_template) + _make_dir(doc_folder) jinja_env = jinja2.Environment() @@ -137,6 +139,15 @@ def generate_report(scenario, base_dir, mongo_url, db_name, doc_folder, LOG.info('The report is written to %s', doc_folder) +def resolve_vars(scenario_template, vars): + jinja_env = jinja2.Environment() + + compiled_template = jinja_env.from_string(scenario_template) + rendered_template = compiled_template.render(vars) + + return rendered_template + + def main(): utils.init_config_and_logging(config.MAIN_OPTS) @@ -144,7 +155,9 @@ def main(): cfg.CONF.scenario, alias_mapper=lambda f: config.SCENARIOS + '%s.yaml' % f) - scenario = utils.read_yaml_file(scenario_file_path) + scenario_raw = utils.read_file(scenario_file_path) + scenario_raw = resolve_vars(scenario_raw, cfg.CONF.vars) + scenario = yaml.safe_load(scenario_raw) base_dir = os.path.dirname(scenario_file_path) generate_report(scenario, base_dir, cfg.CONF.mongo_url, cfg.CONF.mongo_db, diff --git a/performa/modules/omsimulator.py b/performa/modules/omsimulator.py index 1925eda..463ca35 100644 --- a/performa/modules/omsimulator.py +++ b/performa/modules/omsimulator.py @@ -1,9 +1,9 @@ #!/usr/bin/python import copy import os -import tempfile - +import random import signal +import tempfile SERVER_PID = os.path.join(tempfile.gettempdir(), 'performa.oms.pid') SERVER_FILE_NAME = os.path.join(tempfile.gettempdir(), 'performa.oms.srv') @@ -104,16 +104,20 @@ def run(module): client_tool = 'rpc-client' params['python'] = PYTHON + # todo: fix topic support in omsimulator + # params['topic'] = 'performa-%d' % (random.random() * 1000000) params['server_tool'] = server_tool params['client_tool'] = client_tool params['server_file'] = SERVER_FILE_NAME params['client_file'] = CLIENT_FILE_NAME server = ('%(python)s simulator.py ' + # '--topic %(topic)s ' '--url %(url)s ' '--json %(server_file)s ' '%(server_tool)s ') % params client = ('%(python)s simulator.py ' + # '--topic %(topic)s ' '--url=%(url)s ' '--json %(client_file)s ' '-l %(duration)s ' diff --git a/performa/scenarios/mq/omsimulator-threading.rst b/performa/scenarios/mq/omsimulator-threading.rst deleted file mode 100644 index 3d8cd46..0000000 --- a/performa/scenarios/mq/omsimulator-threading.rst +++ /dev/null @@ -1,211 +0,0 @@ -Oslo.messaging simulator report -------------------------------- - -This report shows how many concurrent threads can handle a single -oslo.messaging process. - -Test Case 1: RPC CALL Throughput Test -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -**Message processing** - -Messages are collected at 3 points: ``sent`` - messages sent by the client, -``received`` - messages received by the server, ``round-trip`` - replies -received by the client. Also the number of lost messages is calculated. - -{{''' - title: RPC CALL Message count - axes: - x: threads - y: sent, msg - y2: received, msg - y3: round-trip, msg - y4: lost, msg - chart: line - pipeline: - - { $match: { task: omsimulator, mode: call }} - - { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } }, - sent: { $avg: "$client.count" }, - received: { $avg: "$server.count" }, - round_trip: { $avg: "$round_trip.count" }, - lost: { $avg: { $subtract: ["$client.count", "$round_trip.count"] }} - }} - - { $project: { x: "$_id.threads", - y: "$sent", - y2: "$received", - y3: "$round_trip", - y4: "$lost" - }} - - { $sort: { x: 1 }} -''' | chart -}} - - -**Message throughput, latency and RabbitMQ CPU utilization depending on thread count** - -The chart shows the throughput, latency and CPU utilization by RabbitMQ server -depending on number of concurrent threads. - -{{''' - title: RPC CALL throughput, latency and RabbitMQ CPU utilization depending on thread count - axes: - x: threads - y: sent, msg/sec - y2: received, msg/sec - y3: round-trip, msg/sec - y4: latency, ms - y5: RabbitMQ CPU consumption, % - chart: line - pipeline: - - { $match: { task: omsimulator, mode: call }} - - { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } }, - msg_sent_per_sec: { $avg: { $divide: ["$client.count", "$client.duration"] }}, - msg_received_per_sec: { $avg: { $divide: ["$server.count", "$server.duration"] }}, - msg_round_trip_per_sec: { $avg: { $divide: ["$round_trip.count", "$round_trip.duration"] }}, - latency: { $avg: "$round_trip.latency" }, - rabbit_total: { $avg: "$rabbit_total" } - }} - - { $project: { x: "$_id.threads", - y: "$msg_sent_per_sec", - y2: "$msg_received_per_sec", - y3: "$msg_round_trip_per_sec", - y4: { $multiply: [ "$latency", 1000 ] }, - y5: { $multiply: [ "$rabbit_total", 100 ] } - }} - - { $sort: { x: 1 }} -''' | chart -}} - - -Test Case 2: RPC CAST Throughput Test -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -**Message processing** - -Messages are collected at 2 points: ``sent`` - messages sent by the client -and ``received`` - messages received by the server. Also the number of lost -messages is calculated. - -{{''' - title: RPC CAST Message count - axes: - x: threads - y: sent, msg - y2: received, msg - y3: lost, msg - chart: line - pipeline: - - { $match: { task: omsimulator, mode: cast }} - - { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } }, - sent: { $avg: "$client.count" }, - received: { $avg: "$server.count" }, - lost: { $avg: { $subtract: ["$client.count", "$server.count"] }} - }} - - { $project: { x: "$_id.threads", - y: "$sent", - y2: "$received", - y3: "$lost" - }} - - { $sort: { x: 1 }} -''' | chart -}} - - -**Message throughput, latency and RabbitMQ CPU utilization depending on thread count** - -The chart shows the throughput, latency and CPU utilization by RabbitMQ server -depending on number of concurrent threads. - -{{''' - title: RPC CAST throughput, latency and RabbitMQ CPU utilization depending on thread count - axes: - x: threads - y: sent, msg/sec - y2: received, msg/sec - y3: latency, ms - y4: RabbitMQ CPU consumption, % - chart: line - pipeline: - - { $match: { task: omsimulator, mode: cast }} - - { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } }, - msg_sent_per_sec: { $avg: { $divide: ["$client.count", "$client.duration"] }}, - msg_received_per_sec: { $avg: { $divide: ["$server.count", "$server.duration"] }}, - latency: { $avg: "$server.latency" }, - rabbit_total: { $avg: "$rabbit_total" } - }} - - { $project: { x: "$_id.threads", - y: "$msg_sent_per_sec", - y2: "$msg_received_per_sec", - y3: { $multiply: [ "$latency", 1000 ] }, - y4: { $multiply: [ "$rabbit_total", 100 ] } - }} - - { $sort: { x: 1 }} -''' | chart -}} - - -Test Case 3: Notification Throughput Test -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -**Message processing** - -Messages are collected at 2 points: ``sent`` - messages sent by the client -and ``received`` - messages received by the server. Also the number of lost -messages is calculated. - -{{''' - title: NOTIFY Message count - axes: - x: threads - y: sent, msg - y2: received, msg - l3: lost, msg - chart: line - pipeline: - - { $match: { task: omsimulator, mode: notify }} - - { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } }, - sent: { $avg: "$client.count" }, - received: { $avg: "$server.count" }, - lost: { $avg: { $subtract: ["$client.count", "$server.count"] }} - }} - - { $project: { x: "$_id.threads", - y: "$sent", - y2: "$received", - y3: "$lost" - }} - - { $sort: { x: 1 }} -''' | chart -}} - - -**Message throughput, latency and RabbitMQ CPU utilization depending on thread count** - -The chart shows the throughput, latency and CPU utilization by RabbitMQ server -depending on number of concurrent threads. - -{{''' - title: NOTIFY throughput, latency and RabbitMQ CPU utilization depending on thread count - axes: - x: threads - y: sent, msg/sec - y2: received, msg/sec - y3: latency, ms - y4: RabbitMQ CPU consumption, % - chart: line - pipeline: - - { $match: { task: omsimulator, mode: notify }} - - { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } }, - msg_sent_per_sec: { $avg: { $divide: ["$client.count", "$client.duration"] }}, - msg_received_per_sec: { $avg: { $divide: ["$server.count", "$server.duration"] }}, - latency: { $avg: "$server.latency" }, - rabbit_total: { $avg: "$rabbit_total" } - }} - - { $project: { x: "$_id.threads", - y: "$msg_sent_per_sec", - y2: "$msg_received_per_sec", - y3: { $multiply: [ "$latency", 1000 ] }, - y4: { $multiply: [ "$rabbit_total", 100 ] } - }} - - { $sort: { x: 1 }} -''' | chart -}} diff --git a/performa/scenarios/mq/omsimulator-threading.yaml b/performa/scenarios/mq/omsimulator-threading.yaml index 6032bfd..ec3230b 100644 --- a/performa/scenarios/mq/omsimulator-threading.yaml +++ b/performa/scenarios/mq/omsimulator-threading.yaml @@ -47,7 +47,7 @@ execution: - hosts: {{ tester_hosts }} matrix: - threads: [ 1, 2, 5, 10, 20, 50, 100 ] + threads: [ 1, 2, 5, 10, 20, 50 ] tasks: - omsimulator: mode: call @@ -56,7 +56,7 @@ execution: - hosts: {{ tester_hosts }} matrix: - threads: [ 1, 2, 5, 10, 20, 50, 100 ] + threads: [ 1, 2, 5, 10, 20, 50 ] tasks: - omsimulator: mode: cast @@ -65,7 +65,7 @@ execution: - hosts: {{ tester_hosts }} matrix: - threads: [ 1, 2, 5, 10, 20, 50, 100 ] + threads: [ 1, 2, 5, 10, 20, 50 ] tasks: - omsimulator: mode: notify @@ -89,4 +89,4 @@ aggregation: - { $group: { _id: null, rabbit_sys: { $avg: "$sys" }, rabbit_user: { $avg: "$user" }, rabbit_total: { $avg: { $add: [ "$sys", "$user" ] }} }} report: - template: omsimulator-threading.rst + template: omsimulator.rst diff --git a/performa/scenarios/mq/omsimulator.rst b/performa/scenarios/mq/omsimulator.rst index 695a9b2..de551d1 100644 --- a/performa/scenarios/mq/omsimulator.rst +++ b/performa/scenarios/mq/omsimulator.rst @@ -1,106 +1,217 @@ -OMSimulator Report ------------------- +Oslo.messaging simulator report +------------------------------- -This is the report of execution test plan -:ref:`mq_test_plan` with `OMSimulator`_ tool. +This report is result of `message_queue_performance`_ execution +with `Oslo.messaging Simulator`_ -Results -^^^^^^^ + +Test Case 1: RPC CALL Throughput Test +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ **Message processing** -The chart and table show the number of messages processed by a single process -depending on number of eventlet threads inside of it. Messages are collected -at 3 points: ``sent`` - messages sent by the client, ``received`` - messages -received by the server, ``round-trip`` - replies received by the client. +Messages are collected at 3 points: ``sent`` - messages sent by the client, +``received`` - messages received by the server, ``round-trip`` - replies +received by the client. Also the number of lost messages is calculated. {{''' - title: Throughput + title: RPC CALL Message count axes: x: threads y: sent, msg y2: received, msg y3: round-trip, msg + y4: lost, msg chart: line pipeline: - - { $match: { task: omsimulator, component: client }} + - { $match: { task: omsimulator, mode: call }} - { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } }, - sent: { $avg: "$client.count" }, - received: { $avg: "$server.count" }, - round_trip: { $avg: "$round_trip.count" } + sent: { $sum: "$client.count" }, + received: { $sum: "$server.count" }, + round_trip: { $sum: "$round_trip.count" }, + lost: { $sum: { $subtract: ["$client.count", "$round_trip.count"] }} }} - { $project: { x: "$_id.threads", y: "$sent", y2: "$received", - y3: "$round_trip" + y3: "$round_trip", + y4: "$lost" }} - { $sort: { x: 1 }} ''' | chart }} -**Message throughput, latency depending on thread count** +**Message throughput, latency and RabbitMQ CPU utilization depending on thread count** + +The chart shows the throughput, latency and CPU utilization by RabbitMQ server +depending on number of concurrent threads. {{''' - title: Throughput, latency depending on thread count + title: RPC CALL throughput, latency and RabbitMQ CPU utilization depending on thread count axes: x: threads - y: throughput, msg/sec - y2: latency, ms + y: sent, msg/sec + y2: received, msg/sec + y3: round-trip, msg/sec + y4: latency, ms + y5: RabbitMQ CPU consumption, % chart: line pipeline: - - { $match: { task: omsimulator, component: client }} + - { $match: { task: omsimulator, mode: call }} - { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } }, - msg_sent_per_sec: { $avg: { $divide: ["$count", "$duration"] }}, - latency: { $avg: "$latency" }, - }} - - { $project: { x: "$_id.threads", - y: "$msg_sent_per_sec", - y2: { $multiply: ["$latency", 1000] } - }} - - { $sort: { x: 1 }} -''' | chart -}} - -Messages per second and rabbit CPU consumption depending on threads count: - -{{''' - title: Queries and and CPU util per second - axes: - x: threads - y: queries per sec - y2: rabbit CPU consumption, % - chart: line - pipeline: - - { $match: { task: omsimulator }} - - { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } }, - msg_sent_per_sec: { $avg: { $divide: ["$count", "$duration"] }}, + msg_sent_per_sec: { $avg: { $divide: ["$client.count", "$client.duration"] }}, + msg_received_per_sec: { $avg: { $divide: ["$server.count", "$server.duration"] }}, + msg_round_trip_per_sec: { $avg: { $divide: ["$round_trip.count", "$round_trip.duration"] }}, + latency: { $avg: "$round_trip.latency" }, rabbit_total: { $avg: "$rabbit_total" } }} - { $project: { x: "$_id.threads", y: "$msg_sent_per_sec", - y2: { $multiply: [ "$rabbit_total", 100 ] } + y2: "$msg_received_per_sec", + y3: "$msg_round_trip_per_sec", + y4: { $multiply: [ "$latency", 1000 ] }, + y5: { $multiply: [ "$rabbit_total", 100 ] } }} - { $sort: { x: 1 }} ''' | chart }} +Test Case 2: RPC CAST Throughput Test +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +**Message processing** + +Messages are collected at 2 points: ``sent`` - messages sent by the client +and ``received`` - messages received by the server. Also the number of lost +messages is calculated. + {{''' - title: Latency depending on msg/sec + title: RPC CAST Message count axes: - x: messages per sec - y: latency + x: threads + y: sent, msg + y2: received, msg + y3: lost, msg chart: line pipeline: - - { $match: { task: omsimulator }} + - { $match: { task: omsimulator, mode: cast }} - { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } }, - msg_sent_per_sec: { $avg: { $divide: ["$count", "$duration"] }}, - latency: { $avg: "$latency" } + sent: { $sum: "$client.count" }, + received: { $sum: "$server.count" }, + lost: { $sum: { $subtract: ["$client.count", "$server.count"] }} }} - - { $project: { x: "$msg_sent_per_sec", - y: { $multiply: ["$latency", 1000] } + - { $project: { x: "$_id.threads", + y: "$sent", + y2: "$received", + y3: "$lost" }} - { $sort: { x: 1 }} ''' | chart }} + + +**Message throughput, latency and RabbitMQ CPU utilization depending on thread count** + +The chart shows the throughput, latency and CPU utilization by RabbitMQ server +depending on number of concurrent threads. + +{{''' + title: RPC CAST throughput, latency and RabbitMQ CPU utilization depending on thread count + axes: + x: threads + y: sent, msg/sec + y2: received, msg/sec + y3: latency, ms + y4: RabbitMQ CPU consumption, % + chart: line + pipeline: + - { $match: { task: omsimulator, mode: cast }} + - { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } }, + msg_sent_per_sec: { $avg: { $divide: ["$client.count", "$client.duration"] }}, + msg_received_per_sec: { $avg: { $divide: ["$server.count", "$server.duration"] }}, + latency: { $avg: "$server.latency" }, + rabbit_total: { $avg: "$rabbit_total" } + }} + - { $project: { x: "$_id.threads", + y: "$msg_sent_per_sec", + y2: "$msg_received_per_sec", + y3: { $multiply: [ "$latency", 1000 ] }, + y4: { $multiply: [ "$rabbit_total", 100 ] } + }} + - { $sort: { x: 1 }} +''' | chart +}} + + +Test Case 3: Notification Throughput Test +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +**Message processing** + +Messages are collected at 2 points: ``sent`` - messages sent by the client +and ``received`` - messages received by the server. Also the number of lost +messages is calculated. + +{{''' + title: NOTIFY Message count + axes: + x: threads + y: sent, msg + y2: received, msg + y3: lost, msg + chart: line + pipeline: + - { $match: { task: omsimulator, mode: notify }} + - { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } }, + sent: { $sum: "$client.count" }, + received: { $sum: "$server.count" }, + lost: { $sum: { $subtract: ["$client.count", "$server.count"] }} + }} + - { $project: { x: "$_id.threads", + y: "$sent", + y2: "$received", + y3: "$lost" + }} + - { $sort: { x: 1 }} +''' | chart +}} + + +**Message throughput, latency and RabbitMQ CPU utilization depending on thread count** + +The chart shows the throughput, latency and CPU utilization by RabbitMQ server +depending on number of concurrent threads. + +{{''' + title: NOTIFY throughput, latency and RabbitMQ CPU utilization depending on thread count + axes: + x: threads + y: sent, msg/sec + y2: received, msg/sec + y3: latency, ms + y4: RabbitMQ CPU consumption, % + chart: line + pipeline: + - { $match: { task: omsimulator, mode: notify }} + - { $group: { _id: { threads: { $multiply: [ "$threads", "$host_count" ] } }, + msg_sent_per_sec: { $avg: { $divide: ["$client.count", "$client.duration"] }}, + msg_received_per_sec: { $avg: { $divide: ["$server.count", "$server.duration"] }}, + latency: { $avg: "$server.latency" }, + rabbit_total: { $avg: "$rabbit_total" } + }} + - { $project: { x: "$_id.threads", + y: "$msg_sent_per_sec", + y2: "$msg_received_per_sec", + y3: { $multiply: [ "$latency", 1000 ] }, + y4: { $multiply: [ "$rabbit_total", 100 ] } + }} + - { $sort: { x: 1 }} +''' | chart +}} + +.. references: + +.. _message_queue_performance: http://docs.openstack.org/developer/performance-docs/test_plans/mq/plan.html +.. _Oslo.messaging Simulator: https://github.com/openstack/oslo.messaging/blob/master/tools/simulator.py diff --git a/performa/scenarios/mq/omsimulator.yaml b/performa/scenarios/mq/omsimulator.yaml index 7a236db..8478207 100644 --- a/performa/scenarios/mq/omsimulator.yaml +++ b/performa/scenarios/mq/omsimulator.yaml @@ -4,62 +4,82 @@ description: This scenario uses oslo.messaging simulator tool to execute MQ test plan. parameters: - target: List of hosts were omsimulator will be executed + tester_hosts: List of hosts were omsimulator will be executed + rabbit_hosts: List of hosts were RabbitMQ runs + rabbit_url: RabbitMQ address setup: - - hosts: {{ target }} + hosts: {{ tester_hosts }} tasks: - apt: name=git become: yes -# - name: installing omsimulator -# git: repo=git://git.openstack.org/openstack/oslo.messaging -# dest=/tmp/performa/oslo.messaging -# - name: installing omsimulator -# git: repo=git://git.openstack.org/openstack/oslo.messaging -# dest=/tmp/performa/oslo.messaging -# - command: git fetch https://review.openstack.org/openstack/oslo.messaging refs/changes/91/291191/2 -# args: -# chdir: /tmp/performa/oslo.messaging -# - command: git checkout FETCH_HEAD -# args: -# chdir: /tmp/performa/oslo.messaging -# - apt: name=atop -# become: yes -# - apt: name=daemon -# become: yes + - apt: name=daemon + become: yes + - name: installing omsimulator + git: repo=git://git.openstack.org/openstack/oslo.messaging + dest=/tmp/performa/oslo.messaging + - apt: name=python-dev + become: yes + - apt: name=python-pip + become: yes + - pip: name=virtualenv + become: yes + - pip: requirements=/tmp/performa/oslo.messaging/requirements.txt virtualenv=/tmp/performa/oslo.messaging/.venv + - pip: name=eventlet virtualenv=/tmp/performa/oslo.messaging/.venv + - command: /tmp/performa/oslo.messaging/.venv/bin/python setup.py install + args: + chdir: /tmp/performa/oslo.messaging + - + hosts: {{ rabbit_hosts }} + tasks: + - apt: name=atop + become: yes + - apt: name=daemon + become: yes + execution: - - hosts: {{ target }} + hosts: {{ rabbit_hosts }} tasks: - atop: command=start -# - -# hosts: {{ target }} -# matrix: -# threads: [ 1, 2, 5, 10, 25, 50, 100 ] -# tasks: -# - omsimulator: -# mode: call -# duration: 10 -# url: "rabbit://stackrabbit:swordfish@localhost:5672/" - - hosts: {{ target }} + hosts: {{ tester_hosts }} matrix: - host_count: [ 1 ] - threads: [ 1, 10, 100 ] + host_count: [ 1, 2, 5, 10 ] tasks: - omsimulator: mode: call - duration: 10 + duration: 100 threads: 10 url: {{ rabbit_url }} - - hosts: {{ target }} + hosts: {{ tester_hosts }} + matrix: + host_count: [ 1, 2, 5, 10 ] + tasks: + - omsimulator: + mode: cast + duration: 100 + threads: 10 + url: {{ rabbit_url }} + - + hosts: {{ tester_hosts }} + matrix: + host_count: [ 1, 2, 5, 10 ] + tasks: + - omsimulator: + mode: notify + duration: 100 + threads: 10 + url: {{ rabbit_url }} + - + hosts: {{ rabbit_hosts }} tasks: - atop: command: stop - labels: [ CPU, PRC, PRM ] + labels: [ PRC ] aggregation: - @@ -68,7 +88,7 @@ aggregation: { task: omsimulator } values: pipeline: - - { $match: { task: atop, status: OK, label: PRC, name: beam.smp }} + - { $match: { task: atop, status: OK, label: PRC, name: { $regex: beam.* } }} - { $group: { _id: null, rabbit_sys: { $avg: "$sys" }, rabbit_user: { $avg: "$user" }, rabbit_total: { $avg: { $add: [ "$sys", "$user" ] }} }} report: