Merge "fix prepare for numeric data"

This commit is contained in:
Zuul 2018-09-10 21:17:19 +00:00 committed by Gerrit Code Review
commit f2591e1061
4 changed files with 173 additions and 9 deletions

View File

@ -381,7 +381,10 @@ def get_runs_by_run_metadata_key(run_metadata_key, value):
continue
build_name = run['metadata']['build_name']
if run_at in run_times:
run_times[run_at][build_name].append(run['run_time'])
if build_name in run_times[run_at]:
run_times[run_at][build_name].append(run['run_time'])
else:
run_times[run_at][build_name] = [run['run_time']]
else:
run_times[run_at] = {build_name: [run['run_time']]}
# if there is more than one run with the same run_at time

View File

@ -35,11 +35,11 @@ def get_numeric_data(run_times_time_series, sample_rate):
temp_dict = {}
sample_rate = base.resample_matrix[sample_rate]
for run_at, run in run_times_time_series.items():
build_name, run_time = list(run.items())[0]
if build_name in temp_dict:
temp_dict[build_name][run_at] = run_time
else:
temp_dict[build_name] = {run_at: run_time}
for build_name, run_time in list(run.items()):
if build_name in temp_dict:
temp_dict[build_name][run_at] = run_time
else:
temp_dict[build_name] = {run_at: run_time}
df = pd.DataFrame(temp_dict)
numeric_df = df.resample(sample_rate).mean()
temp_numeric_df = numeric_df.interpolate(method='time', limit=10)

View File

@ -614,13 +614,13 @@ class TestRestAPI(base.TestCase):
response_data = json.loads(res.data.decode('utf-8'))
# numpy.NaN == numpy.NaN result is False, a key error here means the
# dicts are not equal
for project, item in list(expected_response_data['numeric'].items()):
for project, item in expected_response_data['numeric'].items():
for date, run_time in list(item.items()):
if (numpy.isnan(run_time) and
numpy.isnan(response_data['numeric'][project][date])):
del expected_response_data['numeric'][project][date]
del response_data['numeric'][project][date]
self.assertEqual(expected_response_data, response_data)
self.assertDictEqual(expected_response_data, response_data)
api_mock.assert_called_once_with('project',
'openstack/trove',
None,
@ -770,6 +770,85 @@ class TestRestAPI(base.TestCase):
self.maxDiff = None
self.assertDictEqual(expected_response_data, response_data)
@mock.patch('subunit2sql.db.api.get_time_series_runs_by_key_value',
return_value={
timestamp_d1: [{'pass': 1,
'fail': 0,
'skip': 0,
'id': 'abc1',
'run_time': 4.0,
'metadata': {
'build_name':
'tempest-dsvm-neutron-full'}},
{'pass': 10,
'fail': 1,
'skip': 0,
'id': 'abc1',
'run_time': 9.0,
'metadata': {
'build_name':
'tempest-dsvm-neutron-full'}},
{'pass': 2,
'fail': 0,
'skip': 0,
'id': 'abc2',
'run_time': 2.0,
'metadata': {
'build_name':
'openstack-tox-py27-trove'}}],
timestamp_d2: [{'pass': 100,
'fail': 0,
'skip': 0,
'id': 'abc3',
'run_time': 20.0,
'metadata': {
'build_name':
'tempest-dsvm-neutron-full'}}]
})
def test_get_runs_by_project_diff_build_and_same_run_at(self, api_mock):
start_date = timestamp_d1.date().isoformat()
stop_date = timestamp_d2.date().isoformat()
query = ('datetime_resolution=day&start_date={0}&stop_date={1}'
.format(start_date, stop_date))
res = self.app.get('/runs/key/project/trove?{0}'
.format(query))
self.assertEqual(200, res.status_code)
expected_response_data = {
u'data': {
u'timedelta': [
{u'datetime': u'%s' % timestamp_d1.date().isoformat(),
u'job_data': [{u'pass': 1,
u'fail': 0,
u'job_name': u'openstack-tox-py27-trove',
u'mean_run_time': 2.0},
{u'pass': 1,
u'fail': 1,
u'job_name': u'tempest-dsvm-neutron-full',
u'mean_run_time': 6.5}]},
{u'datetime': u'%s' % timestamp_d2.date().isoformat(),
u'job_data': [{u'pass': 1,
u'fail': 0,
u'job_name': u'tempest-dsvm-neutron-full',
u'mean_run_time': 20.0}]}]},
u'numeric': {
u'tempest-dsvm-neutron-full': {
u'%s' % timestamp_d1.isoformat(): 4.0,
u'%s' % timestamp_d2.isoformat(): 20.0},
u'openstack-tox-py27-trove': {
u'%s' % timestamp_d1.isoformat(): 2.0,
u'%s' % timestamp_d2.isoformat(): numpy.NaN}}}
response_data = json.loads(res.data.decode('utf-8'))
self.maxDiff = None
# numpy.NaN == numpy.NaN result is False, a key error here means the
# dicts are not equal
for project, item in expected_response_data['numeric'].items():
for date, run_time in list(item.items()):
if (numpy.isnan(run_time) and
numpy.isnan(response_data['numeric'][project][date])):
del expected_response_data['numeric'][project][date]
del response_data['numeric'][project][date]
self.assertDictEqual(expected_response_data, response_data)
@mock.patch('openstack_health.api._check_db_availability',
return_value=False)
@mock.patch('openstack_health.api._check_er_availability',

View File

@ -63,8 +63,90 @@ class TestRunAggregatorGetNumericData(base.TestCase):
actual = run_aggregator.get_numeric_data({}, 'day')
self.assertEqual(expected, actual)
def test_get_numeric_data_diff_build_name(self):
self.runs[datetime.datetime(2018, 6, 14, 3, 52, 24)][
'openstack-tox-py27-trove'] = 321.304
expected = {
'tempest-dsvm-neutron-full': {
'2018-06-13T00:00:00': 5391.195,
'2018-06-14T00:00:00': 4768.1,
'2018-06-15T00:00:00': np.nan,
'2018-06-16T00:00:00': np.nan,
'2018-06-17T00:00:00': np.nan,
'2018-06-18T00:00:00': 4183.85,
'2018-06-19T00:00:00': 4545.41,
'2018-06-20T00:00:00': 4133.03,
'2018-06-21T00:00:00': np.nan,
'2018-06-22T00:00:00': 5592.295,
'2018-06-23T00:00:00': 6150.95,
'2018-06-24T00:00:00': np.nan,
'2018-06-25T00:00:00': 6047.95
},
'tempest-dsvm-neutron-full-avg': {
'2018-06-13T00:00:00': np.nan,
'2018-06-14T00:00:00': np.nan,
'2018-06-15T00:00:00': np.nan,
'2018-06-16T00:00:00': np.nan,
'2018-06-17T00:00:00': np.nan,
'2018-06-18T00:00:00': np.nan,
'2018-06-19T00:00:00': np.nan,
'2018-06-20T00:00:00': np.nan,
'2018-06-21T00:00:00': np.nan,
'2018-06-22T00:00:00': 4690.44675,
'2018-06-23T00:00:00': 4766.42225,
'2018-06-24T00:00:00': 4899.55725,
'2018-06-25T00:00:00': 5042.148499999999
},
'openstack-tox-py27-trove': {
'2018-06-13T00:00:00': np.nan,
'2018-06-14T00:00:00': 321.304,
'2018-06-15T00:00:00': np.nan,
'2018-06-16T00:00:00': np.nan,
'2018-06-17T00:00:00': np.nan,
'2018-06-18T00:00:00': np.nan,
'2018-06-19T00:00:00': np.nan,
'2018-06-20T00:00:00': np.nan,
'2018-06-21T00:00:00': np.nan,
'2018-06-22T00:00:00': np.nan,
'2018-06-23T00:00:00': np.nan,
'2018-06-24T00:00:00': np.nan,
'2018-06-25T00:00:00': np.nan
},
'openstack-tox-py27-trove-avg': {
'2018-06-13T00:00:00': np.nan,
'2018-06-14T00:00:00': np.nan,
'2018-06-15T00:00:00': np.nan,
'2018-06-16T00:00:00': np.nan,
'2018-06-17T00:00:00': np.nan,
'2018-06-18T00:00:00': np.nan,
'2018-06-19T00:00:00': np.nan,
'2018-06-20T00:00:00': np.nan,
'2018-06-21T00:00:00': np.nan,
'2018-06-22T00:00:00': np.nan,
'2018-06-23T00:00:00': 321.30400000000003,
'2018-06-24T00:00:00': 321.30400000000003,
'2018-06-25T00:00:00': np.nan
}
}
actual = run_aggregator.get_numeric_data(self.runs, 'day')
self.assertItemsEqual(expected, actual)
self.assertItemsEqual(
expected['tempest-dsvm-neutron-full'].keys(),
actual['tempest-dsvm-neutron-full'].keys())
self.assertItemsEqual(
expected['tempest-dsvm-neutron-full-avg'].keys(),
actual['tempest-dsvm-neutron-full-avg'].keys())
# np.nan == np.nan is False, remove the key entries with np.nan values,
# if a key error is thrown then expected does not equal actual.
for key in expected:
for date, run_time in list(expected[key].items()):
if np.isnan(run_time) and np.isnan(actual[key][date]):
del actual[key][date]
del expected[key][date]
self.assertDictEqual(expected, actual)
def test_get_numeric_data(self):
self.maxDiff = None
expected = {
'tempest-dsvm-neutron-full': {
'2018-06-13T00:00:00': 5391.195,