summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorZuul <zuul@review.openstack.org>2018-09-10 21:17:19 +0000
committerGerrit Code Review <review@openstack.org>2018-09-10 21:17:19 +0000
commitf2591e106155da60c8161f78d0f96da6ebef2bac (patch)
treed82019fb1cac972ea92a054ce7e29a414dae0258
parent87ec8c26f0e64e9110ec55e5a31b4aa8b6635d11 (diff)
parenta9bfd78f8dd72bce897d5a6657a70dba09f136ee (diff)
Merge "fix prepare for numeric data"
-rw-r--r--openstack_health/api.py5
-rw-r--r--openstack_health/run_aggregator.py10
-rw-r--r--openstack_health/tests/test_api.py83
-rw-r--r--openstack_health/tests/test_run_aggregator.py84
4 files changed, 173 insertions, 9 deletions
diff --git a/openstack_health/api.py b/openstack_health/api.py
index e498605..1909ab5 100644
--- a/openstack_health/api.py
+++ b/openstack_health/api.py
@@ -381,7 +381,10 @@ def get_runs_by_run_metadata_key(run_metadata_key, value):
381 continue 381 continue
382 build_name = run['metadata']['build_name'] 382 build_name = run['metadata']['build_name']
383 if run_at in run_times: 383 if run_at in run_times:
384 run_times[run_at][build_name].append(run['run_time']) 384 if build_name in run_times[run_at]:
385 run_times[run_at][build_name].append(run['run_time'])
386 else:
387 run_times[run_at][build_name] = [run['run_time']]
385 else: 388 else:
386 run_times[run_at] = {build_name: [run['run_time']]} 389 run_times[run_at] = {build_name: [run['run_time']]}
387 # if there is more than one run with the same run_at time 390 # if there is more than one run with the same run_at time
diff --git a/openstack_health/run_aggregator.py b/openstack_health/run_aggregator.py
index 47eaba4..8d69da1 100644
--- a/openstack_health/run_aggregator.py
+++ b/openstack_health/run_aggregator.py
@@ -35,11 +35,11 @@ def get_numeric_data(run_times_time_series, sample_rate):
35 temp_dict = {} 35 temp_dict = {}
36 sample_rate = base.resample_matrix[sample_rate] 36 sample_rate = base.resample_matrix[sample_rate]
37 for run_at, run in run_times_time_series.items(): 37 for run_at, run in run_times_time_series.items():
38 build_name, run_time = list(run.items())[0] 38 for build_name, run_time in list(run.items()):
39 if build_name in temp_dict: 39 if build_name in temp_dict:
40 temp_dict[build_name][run_at] = run_time 40 temp_dict[build_name][run_at] = run_time
41 else: 41 else:
42 temp_dict[build_name] = {run_at: run_time} 42 temp_dict[build_name] = {run_at: run_time}
43 df = pd.DataFrame(temp_dict) 43 df = pd.DataFrame(temp_dict)
44 numeric_df = df.resample(sample_rate).mean() 44 numeric_df = df.resample(sample_rate).mean()
45 temp_numeric_df = numeric_df.interpolate(method='time', limit=10) 45 temp_numeric_df = numeric_df.interpolate(method='time', limit=10)
diff --git a/openstack_health/tests/test_api.py b/openstack_health/tests/test_api.py
index 106eb99..99b1c2a 100644
--- a/openstack_health/tests/test_api.py
+++ b/openstack_health/tests/test_api.py
@@ -614,13 +614,13 @@ class TestRestAPI(base.TestCase):
614 response_data = json.loads(res.data.decode('utf-8')) 614 response_data = json.loads(res.data.decode('utf-8'))
615 # numpy.NaN == numpy.NaN result is False, a key error here means the 615 # numpy.NaN == numpy.NaN result is False, a key error here means the
616 # dicts are not equal 616 # dicts are not equal
617 for project, item in list(expected_response_data['numeric'].items()): 617 for project, item in expected_response_data['numeric'].items():
618 for date, run_time in list(item.items()): 618 for date, run_time in list(item.items()):
619 if (numpy.isnan(run_time) and 619 if (numpy.isnan(run_time) and
620 numpy.isnan(response_data['numeric'][project][date])): 620 numpy.isnan(response_data['numeric'][project][date])):
621 del expected_response_data['numeric'][project][date] 621 del expected_response_data['numeric'][project][date]
622 del response_data['numeric'][project][date] 622 del response_data['numeric'][project][date]
623 self.assertEqual(expected_response_data, response_data) 623 self.assertDictEqual(expected_response_data, response_data)
624 api_mock.assert_called_once_with('project', 624 api_mock.assert_called_once_with('project',
625 'openstack/trove', 625 'openstack/trove',
626 None, 626 None,
@@ -770,6 +770,85 @@ class TestRestAPI(base.TestCase):
770 self.maxDiff = None 770 self.maxDiff = None
771 self.assertDictEqual(expected_response_data, response_data) 771 self.assertDictEqual(expected_response_data, response_data)
772 772
773 @mock.patch('subunit2sql.db.api.get_time_series_runs_by_key_value',
774 return_value={
775 timestamp_d1: [{'pass': 1,
776 'fail': 0,
777 'skip': 0,
778 'id': 'abc1',
779 'run_time': 4.0,
780 'metadata': {
781 'build_name':
782 'tempest-dsvm-neutron-full'}},
783 {'pass': 10,
784 'fail': 1,
785 'skip': 0,
786 'id': 'abc1',
787 'run_time': 9.0,
788 'metadata': {
789 'build_name':
790 'tempest-dsvm-neutron-full'}},
791 {'pass': 2,
792 'fail': 0,
793 'skip': 0,
794 'id': 'abc2',
795 'run_time': 2.0,
796 'metadata': {
797 'build_name':
798 'openstack-tox-py27-trove'}}],
799 timestamp_d2: [{'pass': 100,
800 'fail': 0,
801 'skip': 0,
802 'id': 'abc3',
803 'run_time': 20.0,
804 'metadata': {
805 'build_name':
806 'tempest-dsvm-neutron-full'}}]
807 })
808 def test_get_runs_by_project_diff_build_and_same_run_at(self, api_mock):
809 start_date = timestamp_d1.date().isoformat()
810 stop_date = timestamp_d2.date().isoformat()
811 query = ('datetime_resolution=day&start_date={0}&stop_date={1}'
812 .format(start_date, stop_date))
813 res = self.app.get('/runs/key/project/trove?{0}'
814 .format(query))
815 self.assertEqual(200, res.status_code)
816 expected_response_data = {
817 u'data': {
818 u'timedelta': [
819 {u'datetime': u'%s' % timestamp_d1.date().isoformat(),
820 u'job_data': [{u'pass': 1,
821 u'fail': 0,
822 u'job_name': u'openstack-tox-py27-trove',
823 u'mean_run_time': 2.0},
824 {u'pass': 1,
825 u'fail': 1,
826 u'job_name': u'tempest-dsvm-neutron-full',
827 u'mean_run_time': 6.5}]},
828 {u'datetime': u'%s' % timestamp_d2.date().isoformat(),
829 u'job_data': [{u'pass': 1,
830 u'fail': 0,
831 u'job_name': u'tempest-dsvm-neutron-full',
832 u'mean_run_time': 20.0}]}]},
833 u'numeric': {
834 u'tempest-dsvm-neutron-full': {
835 u'%s' % timestamp_d1.isoformat(): 4.0,
836 u'%s' % timestamp_d2.isoformat(): 20.0},
837 u'openstack-tox-py27-trove': {
838 u'%s' % timestamp_d1.isoformat(): 2.0,
839 u'%s' % timestamp_d2.isoformat(): numpy.NaN}}}
840 response_data = json.loads(res.data.decode('utf-8'))
841 self.maxDiff = None
842 # numpy.NaN == numpy.NaN result is False, a key error here means the
843 # dicts are not equal
844 for project, item in expected_response_data['numeric'].items():
845 for date, run_time in list(item.items()):
846 if (numpy.isnan(run_time) and
847 numpy.isnan(response_data['numeric'][project][date])):
848 del expected_response_data['numeric'][project][date]
849 del response_data['numeric'][project][date]
850 self.assertDictEqual(expected_response_data, response_data)
851
773 @mock.patch('openstack_health.api._check_db_availability', 852 @mock.patch('openstack_health.api._check_db_availability',
774 return_value=False) 853 return_value=False)
775 @mock.patch('openstack_health.api._check_er_availability', 854 @mock.patch('openstack_health.api._check_er_availability',
diff --git a/openstack_health/tests/test_run_aggregator.py b/openstack_health/tests/test_run_aggregator.py
index 27eb616..0185509 100644
--- a/openstack_health/tests/test_run_aggregator.py
+++ b/openstack_health/tests/test_run_aggregator.py
@@ -63,8 +63,90 @@ class TestRunAggregatorGetNumericData(base.TestCase):
63 actual = run_aggregator.get_numeric_data({}, 'day') 63 actual = run_aggregator.get_numeric_data({}, 'day')
64 self.assertEqual(expected, actual) 64 self.assertEqual(expected, actual)
65 65
66 def test_get_numeric_data_diff_build_name(self):
67 self.runs[datetime.datetime(2018, 6, 14, 3, 52, 24)][
68 'openstack-tox-py27-trove'] = 321.304
69 expected = {
70 'tempest-dsvm-neutron-full': {
71 '2018-06-13T00:00:00': 5391.195,
72 '2018-06-14T00:00:00': 4768.1,
73 '2018-06-15T00:00:00': np.nan,
74 '2018-06-16T00:00:00': np.nan,
75 '2018-06-17T00:00:00': np.nan,
76 '2018-06-18T00:00:00': 4183.85,
77 '2018-06-19T00:00:00': 4545.41,
78 '2018-06-20T00:00:00': 4133.03,
79 '2018-06-21T00:00:00': np.nan,
80 '2018-06-22T00:00:00': 5592.295,
81 '2018-06-23T00:00:00': 6150.95,
82 '2018-06-24T00:00:00': np.nan,
83 '2018-06-25T00:00:00': 6047.95
84 },
85 'tempest-dsvm-neutron-full-avg': {
86 '2018-06-13T00:00:00': np.nan,
87 '2018-06-14T00:00:00': np.nan,
88 '2018-06-15T00:00:00': np.nan,
89 '2018-06-16T00:00:00': np.nan,
90 '2018-06-17T00:00:00': np.nan,
91 '2018-06-18T00:00:00': np.nan,
92 '2018-06-19T00:00:00': np.nan,
93 '2018-06-20T00:00:00': np.nan,
94 '2018-06-21T00:00:00': np.nan,
95 '2018-06-22T00:00:00': 4690.44675,
96 '2018-06-23T00:00:00': 4766.42225,
97 '2018-06-24T00:00:00': 4899.55725,
98 '2018-06-25T00:00:00': 5042.148499999999
99 },
100 'openstack-tox-py27-trove': {
101 '2018-06-13T00:00:00': np.nan,
102 '2018-06-14T00:00:00': 321.304,
103 '2018-06-15T00:00:00': np.nan,
104 '2018-06-16T00:00:00': np.nan,
105 '2018-06-17T00:00:00': np.nan,
106 '2018-06-18T00:00:00': np.nan,
107 '2018-06-19T00:00:00': np.nan,
108 '2018-06-20T00:00:00': np.nan,
109 '2018-06-21T00:00:00': np.nan,
110 '2018-06-22T00:00:00': np.nan,
111 '2018-06-23T00:00:00': np.nan,
112 '2018-06-24T00:00:00': np.nan,
113 '2018-06-25T00:00:00': np.nan
114 },
115 'openstack-tox-py27-trove-avg': {
116 '2018-06-13T00:00:00': np.nan,
117 '2018-06-14T00:00:00': np.nan,
118 '2018-06-15T00:00:00': np.nan,
119 '2018-06-16T00:00:00': np.nan,
120 '2018-06-17T00:00:00': np.nan,
121 '2018-06-18T00:00:00': np.nan,
122 '2018-06-19T00:00:00': np.nan,
123 '2018-06-20T00:00:00': np.nan,
124 '2018-06-21T00:00:00': np.nan,
125 '2018-06-22T00:00:00': np.nan,
126 '2018-06-23T00:00:00': 321.30400000000003,
127 '2018-06-24T00:00:00': 321.30400000000003,
128 '2018-06-25T00:00:00': np.nan
129 }
130
131 }
132 actual = run_aggregator.get_numeric_data(self.runs, 'day')
133 self.assertItemsEqual(expected, actual)
134 self.assertItemsEqual(
135 expected['tempest-dsvm-neutron-full'].keys(),
136 actual['tempest-dsvm-neutron-full'].keys())
137 self.assertItemsEqual(
138 expected['tempest-dsvm-neutron-full-avg'].keys(),
139 actual['tempest-dsvm-neutron-full-avg'].keys())
140 # np.nan == np.nan is False, remove the key entries with np.nan values,
141 # if a key error is thrown then expected does not equal actual.
142 for key in expected:
143 for date, run_time in list(expected[key].items()):
144 if np.isnan(run_time) and np.isnan(actual[key][date]):
145 del actual[key][date]
146 del expected[key][date]
147 self.assertDictEqual(expected, actual)
148
66 def test_get_numeric_data(self): 149 def test_get_numeric_data(self):
67 self.maxDiff = None
68 expected = { 150 expected = {
69 'tempest-dsvm-neutron-full': { 151 'tempest-dsvm-neutron-full': {
70 '2018-06-13T00:00:00': 5391.195, 152 '2018-06-13T00:00:00': 5391.195,