x
/
turbo-hipster
Code Issues Proposed changes

Set benchmarks for innodb counters. 

Use the currently collected innodb counters to decide what we
expect from existing migrations. I've also moved metrics collection
to a cloud db because json was a poor scaling choice.

Change-Id: I2ff0e4b58ac54f53a569ec3e67ad2fa753748dbf
This commit is contained in:
Michael Still 2014-01-16 16:12:46 +11:00
parent e7ee50e0c0
commit c972d2d1d7
14 changed files with 225 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
datasets/datasets_devstack_131007/131007_devstack_export/config.json
View File

@ -1,10 +1,16 @@
{
"Innodb_rows_read": {
"default": 1000
},
"XInnodb_rows_changed": {
"default": 1000
},
"database": "nova_dataset_131007_devstack",
"db_pass": "tester",
"db_user": "nova",
"logging_conf": "logging.conf",
"maximum_migration_times": {
"default": 30
"default": 60
},
"project": "openstack/nova",
"seed_data": "nova.sql",

8
datasets/datasets_devstack_131007/131007_devstack_export/input.json
View File

@ -7,6 +7,12 @@
"seed_data": "nova.sql",
"logging_conf": "logging.conf",
"maximum_migration_times": {
"default": 30
"default": 60
},
"XInnodb_rows_changed": {
"default": 1000
},
"Innodb_rows_read": {
"default": 1000
}
}

9
datasets/datasets_devstack_150/datasets_devstack_150/config.json
View File

@ -1,10 +1,17 @@
{
"Innodb_rows_read": {
"default": 1000
},
"XInnodb_rows_changed": {
"default": 1000
},
"database": "datasets_devstack_150",
"db_pass": "tester",
"db_user": "nova",
"logging_conf": "logging.conf",
"maximum_migration_times": {
"default": 30
"151->152": 67.0,
"default": 60
},
"project": "openstack/nova",
"seed_data": "nova.sql",

8
datasets/datasets_devstack_150/datasets_devstack_150/input.json
View File

@ -7,7 +7,13 @@
"seed_data": "nova.sql",
"logging_conf": "logging.conf",
"maximum_migration_times": {
"default": 30
"default": 60
},
"XInnodb_rows_changed": {
"default": 1000
},
"Innodb_rows_read": {
"default": 1000
}
}

11
datasets/datasets_trivial_500/nova_trivial_500/config.json
View File

@ -1,11 +1,18 @@
{
"Innodb_rows_read": {
"default": 1000
},
"XInnodb_rows_changed": {
"default": 1000
},
"database": "nova_dataset_trivial_500",
"db_pass": "tester",
"db_user": "nova",
"logging_conf": "logging.conf",
"maximum_migration_times": {
"138": 42.0,
"default": 30
"151->152": 84.0,
"152->151": 103.0,
"default": 60
},
"project": "openstack/nova",
"seed_data": "nova_trivial_500.sql",

8
datasets/datasets_trivial_500/nova_trivial_500/input.json
View File

@ -7,7 +7,13 @@
"seed_data": "nova_trivial_500.sql",
"logging_conf": "logging.conf",
"maximum_migration_times": {
"default": 30
"default": 60
},
"XInnodb_rows_changed": {
"default": 1000
},
"Innodb_rows_read": {
"default": 1000
}
}

13
datasets/datasets_trivial_6000/nova_trivial_6000/config.json
View File

@ -1,11 +1,20 @@
{
"Innodb_rows_read": {
"default": 1000
},
"XInnodb_rows_changed": {
"default": 1000
},
"database": "nova_dataset_trivial_6000",
"db_pass": "tester",
"db_user": "nova",
"logging_conf": "logging.conf",
"maximum_migration_times": {
"152": 74.0,
"default": 30
"151->152": 159.0,
"152->151": 195.0,
"184->185": 66.0,
"186->185": 144.0,
"default": 60
},
"project": "openstack/nova",
"seed_data": "nova_trivial_6000.sql",

8
datasets/datasets_trivial_6000/nova_trivial_6000/input.json
View File

@ -7,7 +7,13 @@
"seed_data": "nova_trivial_6000.sql",
"logging_conf": "logging.conf",
"maximum_migration_times": {
"default": 30
"default": 60
},
"XInnodb_rows_changed": {
"default": 1000
},
"Innodb_rows_read": {
"default": 1000
}
}

51
datasets/datasets_user_001/user_001/config.json
View File

@ -1,18 +1,49 @@
{
"Innodb_rows_read": {
"148->149": 110000,
"151->152": 3470000,
"159->160": 200000,
"160->161": 390000,
"202->203": 260000,
"205->206": 140000,
"215->216": 930000,
"default": 100000
},
"XInnodb_rows_changed": {
"148->149": 110000,
"151->152": 3200000,
"184->185": 140000,
"193->194": 150000,
"202->203": 520000,
"203->204": 260000,
"205->206": 190000,
"215->216": 260000,
"229->230": 140000,
"default": 100000
},
"database": "nova_datasets_user_001",
"db_pass": "tester",
"db_user": "nova",
"logging_conf": "logging.conf",
"maximum_migration_times": {
"135": 62.0,
"138": 44.0,
"149": 87.0,
"152": 241.0,
"159": 86.0,
"205": 51.0,
"206": 63.0,
"216": 102.0,
"230": 53.0,
"134->135": 116.0,
"135->134": 97.0,
"137->138": 85.0,
"138->137": 100.0,
"148->149": 135.0,
"149->148": 158.0,
"151->152": 333.0,
"152->151": 330.0,
"158->159": 136.0,
"159->158": 168.0,
"186->185": 569.0,
"204->205": 97.0,
"205->204": 98.0,
"205->206": 116.0,
"206->205": 106.0,
"215->216": 137.0,
"229->230": 122.0,
"230->229": 84.0,
"_138_bugs": [
1263835
],
@ -22,7 +53,7 @@
"_205_bugs": [
1263868
],
"default": 30
"default": 60
},
"project": "openstack/nova",
"seed_data": "nova_user_001.sql",

17
datasets/datasets_user_001/user_001/input.json
View File

@ -7,17 +7,16 @@
"seed_data": "nova_user_001.sql",
"logging_conf": "logging.conf",
"maximum_migration_times": {
"default": 30,
"135": 120,
"138": 180,
"default": 60,
"_138_bugs": [1263835],
"149": 240,
"_149_bugs": [1263836],
"152": 300,
"159": 120,
"205": 120,
"_205_bugs": [1263868],
"216": 180
"_205_bugs": [1263868]
},
"XInnodb_rows_changed": {
"default": 100000
},
"Innodb_rows_read": {
"default": 100000
}
}

8
datasets/datasets_user_002/user_002/config.json
View File

@ -1,10 +1,16 @@
{
"Innodb_rows_read": {
"default": 100000
},
"XInnodb_rows_changed": {
"default": 100000
},
"database": "nova_dataset_user_002",
"db_pass": "tester",
"db_user": "nova",
"logging_conf": "logging.conf",
"maximum_migration_times": {
"default": 30
"default": 60
},
"project": "openstack/nova",
"seed_data": "nova_user_002.sql",

8
datasets/datasets_user_002/user_002/input.json
View File

@ -7,7 +7,13 @@
"seed_data": "nova_user_002.sql",
"logging_conf": "logging.conf",
"maximum_migration_times": {
"default": 30
"default": 60
},
"XInnodb_rows_changed": {
"default": 100000
},
"Innodb_rows_read": {
"default": 100000
}
}

33
turbo_hipster/cmd/analyse_historical.py
View File

@ -59,6 +59,7 @@ def main():
# Open the results database
db = MySQLdb.connect(host=config['results']['host'],
port=config['results'].get('port', 3306),
user=config['results']['username'],
passwd=config['results']['password'],
db=config['results']['database'])
@ -83,16 +84,28 @@ def main():
if not 'duration' in migration:
continue
cursor.execute('insert ignore into summary'
'(path, parsed_at, engine, dataset, '
'migration, duration, stats_json) '
'values("%s", now(), "%s", '
'"%s", "%s", %d, "%s");'
% (item['name'], engine, dataset,
'%s->%s' % (migration['from'],
migration['to']),
migration['duration'],
migration['stats']))
if migration['stats']:
cursor.execute('insert ignore into summary'
'(path, parsed_at, engine, dataset, '
'migration, duration, stats_json) '
'values(%s, now(), %s, '
'%s, %s, %s, %s);',
(item['name'], engine, dataset,
'%s->%s' % (migration['from'],
migration['to']),
migration['duration'],
json.dumps(migration['stats'])))
else:
cursor.execute('insert ignore into summary'
'(path, parsed_at, engine, dataset, '
'migration, duration, stats_json) '
'values(%s, now(), %s, '
'%s, %s, %s, NULL);',
(item['name'], engine, dataset,
'%s->%s' % (migration['from'],
migration['to']),
migration['duration']))
cursor.execute('commit;')
items = connection.get_container(swift_config['container'],

135
turbo_hipster/cmd/report_historical.py
View File

@ -17,7 +17,7 @@
import json
import math
import numpy
import MySQLdb
import os
import sys
@ -29,35 +29,63 @@ def main():
def process_dataset(dataset):
with open('results.json') as f:
results = json.loads(f.read())
with open('/etc/turbo-hipster/config.json', 'r') as config_stream:
config = json.load(config_stream)
db = MySQLdb.connect(host=config['results']['host'],
port=config['results'].get('port', 3306),
user=config['results']['username'],
passwd=config['results']['password'],
db=config['results']['database'])
cursor = db.cursor(MySQLdb.cursors.DictCursor)
migrations = {}
all_times = {}
stats_summary = {}
for engine in ['mysql', 'percona']:
print
print 'Dataset: %s' % dataset
print 'Engine: %s' % engine
print
print '%s, %s' % (dataset, engine)
cursor.execute('select distinct(migration) from summary where '
'engine="%s" and dataset="%s" order by migration;'
% (engine, dataset))
migrations_list = []
for row in cursor:
migrations_list.append(row['migration'])
for migration in sorted(results[engine][dataset]):
times = []
for migration in migrations_list:
all_times.setdefault(migration, [])
for time in results[engine][dataset][migration]:
for i in range(results[engine][dataset][migration][time]):
times.append(int(time))
all_times[migration].append(int(time))
times = sorted(times)
emit_summary(engine, times, migrations, migration)
cursor.execute('select distinct(duration), count(*) from summary '
'where engine="%s" and dataset="%s" and '
'migration="%s" group by duration;'
% (engine, dataset, migration))
for row in cursor:
for i in range(row['count(*)']):
all_times[migration].append(row['duration'])
print
print 'Dataset: %s' % dataset
print 'Engine: combined'
print
for migration in sorted(all_times.keys()):
emit_summary('combined', all_times[migration], migrations, migration)
cursor.execute('select stats_json from summary where engine="%s" '
'and dataset="%s" and migration="%s" and '
'not (stats_json = "{}");'
% (engine, dataset, migration))
for row in cursor:
stats = json.loads(row['stats_json'])
for key in stats:
stats_summary.setdefault(migration, {})
stats_summary[migration].setdefault(key, {})
stats_summary[migration][key].setdefault(stats[key], 0)
stats_summary[migration][key][stats[key]] += 1
# Composed stats
rows_changed = 0
for key in ['Innodb_rows_updated',
'Innodb_rows_inserted',
'Innodb_rows_deleted']:
rows_changed += stats.get(key, 0)
stats_summary[migration].setdefault('XInnodb_rows_changed', {})
stats_summary[migration]['XInnodb_rows_changed'].setdefault(
rows_changed, 0)
stats_summary[migration]['XInnodb_rows_changed'][rows_changed]\
+= 1
with open('results.txt', 'w') as f:
f.write('Migration,mysql,percona\n')
@ -75,10 +103,33 @@ def process_dataset(dataset):
config = json.loads(f.read())
for migration in sorted(all_times.keys()):
minimum, mean, maximum, stddev = analyse(all_times[migration])
recommend = mean + 2 * stddev
if recommend > 30.0:
config['maximum_migration_times'][migration] = math.ceil(recommend)
# Timing
config_max = config['maximum_migration_times']['default']
l = len(all_times[migration])
if l > 10:
sorted_all_times = sorted(all_times[migration])
one_percent = int(math.ceil(l / 100))
recommend = sorted_all_times[-one_percent] + 30
if recommend > config_max:
config['maximum_migration_times'][migration] = \
math.ceil(recommend)
# Innodb stats
if not migration in stats_summary:
continue
for stats_key in ['XInnodb_rows_changed', 'Innodb_rows_read']:
config_max = config[stats_key]['default']
values = []
results = stats_summary[migration].get(stats_key, {})
for result in results:
values.append(result)
max_value = max(values)
rounding = max_value % 10000
if max_value > config_max:
config[stats_key][migration] = max_value + (10000 - rounding)
with open(os.path.join(config_path, 'config.json'), 'w') as f:
f.write(json.dumps(config, indent=4, sort_keys=True))
@ -94,40 +145,6 @@ def omg_hard_to_predict_names(dataset):
return dataset
def analyse(times):
np_times = numpy.array(times)
minimum = np_times.min()
mean = np_times.mean()
maximum = np_times.max()
stddev = np_times.std()
return minimum, mean, maximum, stddev
def emit_summary(engine, times, migrations, migration):
minimum, mean, maximum, stddev = analyse(times)
failed_threshold = int(max(30.0, mean + stddev * 2))
failed = 0
for time in times:
if time > failed_threshold:
failed += 1
migrations.setdefault(migration, {})
migrations[migration][engine] = ('%.02f;%0.2f;%.02f'
% (mean - 2 * stddev,
mean,
mean + 2 * stddev))
if failed_threshold != 30 or failed > 0:
print ('%s: Values range from %s to %s seconds. %d values. '
'Mean is %.02f, stddev is %.02f.\n '
'Recommend max of %d. With this value %.02f%% of tests '
'would have failed.'
% (migration, minimum, maximum,
len(times), mean, stddev, failed_threshold,
failed * 100.0 / len(times)))
if __name__ == '__main__':
sys.path.insert(0, os.path.abspath(
os.path.join(os.path.dirname(__file__), '../')))