Enrich sysbench-oltp scenario

* Collect latency stats
* Update report template

Change-Id: Ie29bc0a0b25261f0bdf966731aab8eb22c6880ba

performa/modules/sysbench_oltp.py

@@ -10,12 +10,21 @@ TEST_STATS = re.compile(
     '\s+total:\s+(?P<queries_total>\d+).*\n',
     flags=re.MULTILINE | re.DOTALL
 )
+LATENCY_STATS = re.compile(
+    '\s+per-request statistics:\s*\n'
+    '\s+min:\s+(?P<latency_min>[\d\.]+)ms\s*\n'
+    '\s+avg:\s+(?P<latency_avg>[\d\.]+)ms\s*\n'
+    '\s+max:\s+(?P<latency_max>[\d\.]+)ms\s*\n'
+    '\s+approx.+:\s+(?P<latency_95p>[\d\.]+)ms\s*\n',
+    flags=re.MULTILINE | re.DOTALL
+)
 PATTERNS = [
     r'sysbench (?P<version>[\d\.]+)',
     TEST_STATS,
     r'\s+transactions:\s+(?P<transactions>\d+).*\n',
     r'\s+deadlocks:\s+(?P<deadlocks>\d+).*\n',
     r'\s+total time:\s+(?P<duration>[\d\.]+).*\n',
+    LATENCY_STATS,
 ]
 TRANSFORM_FIELDS = {
     'queries_read': int,
@@ -25,6 +34,10 @@ TRANSFORM_FIELDS = {
     'duration': float,
     'transactions': int,
     'deadlocks': int,
+    'latency_min': float,
+    'latency_avg': float,
+    'latency_max': float,
+    'latency_95p': float,
 }
 
 
@@ -49,6 +62,7 @@ def main():
             threads=dict(type='int', default=10),
             duration=dict(type='int', default=10),
             mysql_host=dict(default='localhost'),
+            mysql_port=dict(type='int', default=3306),
             mysql_db=dict(default='sbtest'),
             oltp_table_name=dict(default='sbtest'),
             oltp_table_size=dict(type='int', default=100000),
@@ -63,6 +77,7 @@ def main():
            '--max-time=%(duration)s '
            '--max-requests=0 '
            '--mysql-host=%(mysql_host)s '
+           '--mysql-port=%(mysql_port)s '
            '--mysql-db=%(mysql_db)s '
            '--oltp-table-name=%(oltp_table_name)s '
            '--oltp-table-size=%(oltp_table_size)s '

performa/scenarios/db/sysbench.rst

@@ -1,56 +1,96 @@
 Sysbench Report
 ---------------
 
-This is the report of execution test plan
-:ref:`sql_db_test_plan` with `Sysbench`_ tool.
+This scenario is executed with `Sysbench`_ tool. There is one instance of
+tool per tester node, each running in N threads.
 
-Results
-^^^^^^^
+Throughput
+^^^^^^^^^^
 
-**Queries per second depending on threads count**
+The following chart shows the number of queries, read queries and transactions
+depending on total thread count.
 
 {{'''
-    title: Queries per second
+    title: Throughput
     axes:
       x: threads
-      y: queries per sec
+      y1: queries per sec
       y2: read queries per sec
+      y3: transactions per sec
     chart: line
     pipeline:
-    - $match: { task: sysbench_oltp, status: OK }
-    - $group: { _id: { threads: "$threads" },
-                queries_total_per_sec: { $avg: { $divide: ["$queries_total", "$duration"] }},
-                queries_read_per_sec: { $avg: { $divide: ["$queries_read", "$duration"] }}
-              }
+    - $match: { task: sysbench_oltp }
+    - $group:
+        _id: { threads: { $multiply: [ "$threads", "$host_count" ] } }
+        queries_total_per_sec: { $sum: { $divide: ["$queries_total", "$duration"] }}
+        queries_read_per_sec: { $sum: { $divide: ["$queries_read", "$duration"] }}
+        transactions_per_sec: { $sum: { $divide: ["$transactions", "$duration"] }}
     - $project:
         x: "$_id.threads"
-        y: "$queries_total_per_sec"
+        y1: "$queries_total_per_sec"
         y2: "$queries_read_per_sec"
+        y3: "$transactions_per_sec"
 ''' | chart_and_table
 }}
 
-**Queries per second and mysqld CPU consumption depending on threads count**
+
+Throughput and server CPU consumption
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The following chart shows how DB server CPU consumption depends on number
+of concurrent threads and throughput.
 
 {{'''
-    title: Queries and and CPU util per second
+    title: CPU consumption
     axes:
       x: threads
-      y: queries per sec
-      y2: mysqld CPU consumption, %
+      y1: queries per sec
+      y2: CPU, %
     chart: line
     pipeline:
-    - $match: { task: sysbench_oltp, status: OK }
-    - $group: { _id: { threads: "$threads" },
-                queries_total_per_sec: { $avg: { $divide: ["$queries_total", "$duration"] }},
-                mysqld_total: { $avg: "$mysqld_total" }
-              }
+    - $match: { task: sysbench_oltp }
+    - $group:
+        _id: { threads: { $multiply: [ "$threads", "$host_count" ] } }
+        queries_total_per_sec: { $sum: { $divide: ["$queries_total", "$duration"] }}
+        mysqld_total: { $avg: "$mysqld_total" }
     - $project:
         x: "$_id.threads"
-        y: "$queries_total_per_sec"
+        y1: "$queries_total_per_sec"
         y2: { $multiply: [ "$mysqld_total", 100 ] }
 ''' | chart_and_table
 }}
 
+
+Operation latency
+^^^^^^^^^^^^^^^^^
+
+The following chart shows how operation latency depends on number of
+concurrent threads.
+
+{{'''
+    title: Latency
+    axes:
+      x: threads
+      y1: min latency, ms
+      y2: avg latency, ms
+      y3: max latency, ms
+    chart: line
+    pipeline:
+    - $match: { task: sysbench_oltp }
+    - $group:
+        _id: { threads: { $multiply: [ "$threads", "$host_count" ] } }
+        latency_min: { $min: "$latency_min" }
+        latency_avg: { $avg: "$latency_avg" }
+        latency_max: { $max: "$latency_max" }
+    - $project:
+        x: "$_id.threads"
+        y1: "$latency_min"
+        y2: "$latency_avg"
+        y3: "$latency_max"
+''' | chart_and_table
+}}
+
+
 .. references:
 
 .. _Sysbench: https://github.com/akopytov/sysbench

performa/scenarios/db/sysbench.yaml

@@ -6,6 +6,8 @@ description:
 parameters:
   tester_hosts: List of hosts were omsimulator will be executed
   mysql_hosts: List of hosts were MySQL runs
+  mysql_endpoint: Address of MySQL endpoint
+  mysql_port: MySQL port number
 
 setup:
   -
@@ -30,11 +32,12 @@ execution:
   -
     hosts: {{ tester_hosts }}
     matrix:
-      threads: [ 10, 20, 50, 70, 100, 150, 200, 300 ]
+      threads: [ 5, 10, 15, 20, 30, 40, 50 ]
     tasks:
     - sysbench_oltp:
         duration: 60
-        mysql_host: localhost
+        mysql_host: {{ mysql_endpoint }}
+        mysql_port: {{ mysql_port }}
         mysql_db: sbtest
   -
     hosts: {{ mysql_hosts }}
@@ -51,7 +54,8 @@ aggregation:
       values:
         pipeline:
         - { $match: { task: atop, status: OK, label: PRC, name: mysqld }}
-        - { $group: { _id: null, mysqld_sys: { $avg: "$sys" }, mysqld_user: { $avg: "$user" }, mysqld_total: { $avg: { $add: [ "$sys", "$user" ] }} }}
+        - { $group: { _id: "$host", mysqld_sys: { $avg: "$sys" }, mysqld_user: { $avg: "$user" }, mysqld_total: { $avg: { $add: [ "$sys", "$user" ] }} }}
+        - { $group: { _id: null, mysqld_sys: { $sum: "$mysqld_sys"}, mysqld_user: { $sum: "$mysqld_user" }, mysqld_total: { $sum: "$mysqld_total" }}}
 
 report:
   template: sysbench.rst

performa/tests/omsimulator_sample.txt

@@ -1,5 +0,0 @@
-2016-02-29 10:33:51,044 INFO root Preparing 1 messages
-2016-02-29 10:33:51,046 INFO root Messages has been prepared
-2016-02-29 10:34:01,062 INFO root 5313 messages were sent for 10 seconds. Bandwidth was 530 msg/sec
-2016-02-29 10:34:01,062 INFO root 14897652 bytes were sent for 10 seconds. Bandwidth is 1487420 b/s
-2016-02-29 10:34:01,062 INFO root calls finished, wait 0 seconds

performa/tests/test_sysbench.py

@@ -72,6 +72,10 @@ class TestSysbench(testtools.TestCase):
             'transactions': 665,
             'deadlocks': 0,
             'duration': 60.8074,
+            'latency_min': 876.31,
+            'latency_avg': 1816.63,
+            'latency_max': 3792.73,
+            'latency_95p': 2886.19,
         }
 
         self.assertEqual(expected, sysbench.parse_sysbench_oltp(OLTP_OUTPUT))