From efc880799b8a6b7292ec6560b0d25154b68c85c6 Mon Sep 17 00:00:00 2001 From: Lingxian Kong Date: Fri, 6 Jul 2018 14:43:10 +1200 Subject: [PATCH] Get last collection time from valid projects If some projects are ignored, we should not take them into consideration when calculating the last collection time. Change-Id: I44393101fd655ab7dea542493c53245616ec3c4a --- distil/service/collector.py | 4 +- distil/tests/unit/service/test_collector.py | 49 ++++++++++++++++----- 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/distil/service/collector.py b/distil/service/collector.py index 452e0be..8bdbf1d 100644 --- a/distil/service/collector.py +++ b/distil/service/collector.py @@ -118,11 +118,11 @@ class CollectorService(service.Service): return projects = openstack.get_projects() - project_ids = [p['id'] for p in projects] valid_projects = filter_projects(projects) + project_ids = [p['id'] for p in valid_projects] # For new created project, we use the earliest last collection time - # among existing projects as the start time. + # among existing valid projects as the start time. last_collect = db_api.get_last_collect(project_ids).last_collected end = datetime.utcnow().replace(minute=0, second=0, microsecond=0) diff --git a/distil/tests/unit/service/test_collector.py b/distil/tests/unit/service/test_collector.py index c68153e..816908b 100644 --- a/distil/tests/unit/service/test_collector.py +++ b/distil/tests/unit/service/test_collector.py @@ -108,17 +108,14 @@ class CollectorTest(base.DistilWithDbTestCase): @mock.patch('distil.common.openstack.get_projects') def test_last_collect_new_project(self, mock_get_projects, mock_cclient, mock_collect_usage): - self.override_config('collector', include_tenants=['project_3']) - - # Assume project_3 is a new project that doesn't exist in distil db. + # Assume project_2 is a new project that doesn't exist in distil db. mock_get_projects.return_value = [ {'id': '111', 'name': 'project_1', 'description': ''}, {'id': '222', 'name': 'project_2', 'description': ''}, - {'id': '333', 'name': 'project_3', 'description': ''}, ] - # Insert 3 projects in the database, including one project which is - # not in keystone. + # Insert project_0 and project_1 in the database, project_0 is not in + # keystone anymore. project_0_collect = datetime(2017, 5, 17, 19) db_api.project_add( { @@ -137,22 +134,54 @@ class CollectorTest(base.DistilWithDbTestCase): }, project_1_collect ) - project_2_collect = datetime(2017, 5, 17, 21) + + svc = collector.CollectorService() + svc.collect_usage() + + self.assertEqual(2, mock_collect_usage.call_count) + mock_collect_usage.assert_called_with( + {'id': '222', 'name': 'project_2', 'description': ''}, + [(project_1_collect, project_1_collect + timedelta(hours=1))] + ) + + @mock.patch( + 'distil.collector.ceilometer.CeilometerCollector.collect_usage') + @mock.patch('distil.common.openstack.get_ceilometer_client') + @mock.patch('distil.common.openstack.get_projects') + def test_last_collect_ignore_project(self, mock_get_projects, mock_cclient, + mock_collect_usage): + self.override_config('collector', ignore_tenants=['project_2']) + + mock_get_projects.return_value = [ + {'id': '111', 'name': 'project_1', 'description': ''}, + {'id': '222', 'name': 'project_2', 'description': ''}, + ] + + project1_time = datetime(2017, 5, 17, 20) + db_api.project_add( + { + 'id': '111', + 'name': 'project_1', + 'description': '', + }, + project1_time + ) + project2_time = datetime(2017, 5, 17, 19) db_api.project_add( { 'id': '222', 'name': 'project_2', 'description': '', }, - project_2_collect + project2_time ) svc = collector.CollectorService() svc.collect_usage() mock_collect_usage.assert_called_once_with( - {'id': '333', 'name': 'project_3', 'description': ''}, - [(project_1_collect, project_1_collect + timedelta(hours=1))] + {'id': '111', 'name': 'project_1', 'description': ''}, + [(project1_time, project1_time + timedelta(hours=1))] ) @mock.patch('distil.common.openstack.get_ceilometer_client')