From d906c733a231c4c7ea244c76b3f9b1e7b634b756 Mon Sep 17 00:00:00 2001 From: Lingxian Kong Date: Mon, 10 Jul 2017 15:01:21 +1200 Subject: [PATCH] Support project order for collection The project ids returned from Keystone is in ascending order which may lead to some projects will never get updated if some unexcepted error happens during collection. This patch adds a config option to define the prefered collection order, currently, 'ascending', 'descending' and 'random' are supported. Change-Id: Ife97e73fe2ae4533ef4baf1f935e46579902719e --- distil/config.py | 6 +- distil/service/collector.py | 12 +++ distil/tests/unit/service/test_collector.py | 97 +++++++++++++++++++++ 3 files changed, 114 insertions(+), 1 deletion(-) diff --git a/distil/config.py b/distil/config.py index dc663b7..f63c3b1 100644 --- a/distil/config.py +++ b/distil/config.py @@ -68,7 +68,11 @@ COLLECTOR_OPTS = [ help=('The earlist starting time for new tenant.')), cfg.StrOpt('partitioning_suffix', help=('Collector partitioning group suffix. It is used when ' - 'running multiple collectors in favor of lock.')) + 'running multiple collectors in favor of lock.')), + cfg.StrOpt('project_order', default='ascending', + choices=['ascending', 'descending', 'random'], + help=('The order of project IDs to do usage collection. ' + 'Default is ascending.')) ] ODOO_OPTS = [ diff --git a/distil/service/collector.py b/distil/service/collector.py index 49eacf8..d6c9153 100644 --- a/distil/service/collector.py +++ b/distil/service/collector.py @@ -13,6 +13,7 @@ # under the License. from datetime import datetime +from random import shuffle from oslo_config import cfg from oslo_log import log as logging @@ -96,6 +97,16 @@ class CollectorService(service.Service): super(CollectorService, self).reset() logging.setup(CONF, 'distil-collector') + def _get_projects_by_order(self, projects): + if CONF.collector.project_order == 'ascending': + return projects + elif CONF.collector.project_order == 'descending': + projects.reverse() + return projects + elif CONF.collector.project_order == 'random': + shuffle(projects) + return projects + def collect_usage(self): LOG.info("Starting to collect usage...") @@ -110,6 +121,7 @@ class CollectorService(service.Service): end = datetime.utcnow().replace(minute=0, second=0, microsecond=0) count = 0 + valid_projects = self._get_projects_by_order(valid_projects) for project in valid_projects: # Check if the project is being processed by other collector # instance. If no, will get a lock and continue processing, diff --git a/distil/tests/unit/service/test_collector.py b/distil/tests/unit/service/test_collector.py index cc10599..e9ae683 100644 --- a/distil/tests/unit/service/test_collector.py +++ b/distil/tests/unit/service/test_collector.py @@ -154,3 +154,100 @@ class CollectorTest(base.DistilWithDbTestCase): project_1_collect, end ) + + @mock.patch('distil.db.api.get_project_locks') + @mock.patch('distil.common.openstack.get_ceilometer_client') + @mock.patch('distil.common.openstack.get_projects') + def test_project_order_ascending(self, mock_get_projects, mock_cclient, + mock_getlocks): + mock_get_projects.return_value = [ + {'id': '111', 'name': 'project_1', 'description': ''}, + {'id': '222', 'name': 'project_2', 'description': ''}, + {'id': '333', 'name': 'project_3', 'description': ''}, + {'id': '444', 'name': 'project_4', 'description': ''}, + ] + + # Insert a project in the database in order to get last_collect time. + db_api.project_add( + { + 'id': '111', + 'name': 'project_1', + 'description': '', + }, + datetime(2017, 5, 17, 19) + ) + + svc = collector.CollectorService() + svc.collect_usage() + + self.assertEqual( + [mock.call('111'), mock.call('222'), mock.call('333'), + mock.call('444')], + mock_getlocks.call_args_list + ) + + @mock.patch('distil.db.api.get_project_locks') + @mock.patch('distil.common.openstack.get_ceilometer_client') + @mock.patch('distil.common.openstack.get_projects') + def test_project_order_descending(self, mock_get_projects, mock_cclient, + mock_getlocks): + self.override_config('collector', project_order='descending') + + mock_get_projects.return_value = [ + {'id': '111', 'name': 'project_1', 'description': ''}, + {'id': '222', 'name': 'project_2', 'description': ''}, + {'id': '333', 'name': 'project_3', 'description': ''}, + {'id': '444', 'name': 'project_4', 'description': ''}, + ] + + # Insert a project in the database in order to get last_collect time. + db_api.project_add( + { + 'id': '111', + 'name': 'project_1', + 'description': '', + }, + datetime(2017, 5, 17, 19) + ) + + svc = collector.CollectorService() + svc.collect_usage() + + self.assertEqual( + [mock.call('444'), mock.call('333'), mock.call('222'), + mock.call('111')], + mock_getlocks.call_args_list + ) + + @mock.patch('distil.db.api.get_project_locks') + @mock.patch('distil.common.openstack.get_ceilometer_client') + @mock.patch('distil.common.openstack.get_projects') + def test_project_order_random(self, mock_get_projects, mock_cclient, + mock_getlocks): + self.override_config('collector', project_order='random') + + mock_get_projects.return_value = [ + {'id': '111', 'name': 'project_1', 'description': ''}, + {'id': '222', 'name': 'project_2', 'description': ''}, + {'id': '333', 'name': 'project_3', 'description': ''}, + {'id': '444', 'name': 'project_4', 'description': ''}, + ] + + # Insert a project in the database in order to get last_collect time. + db_api.project_add( + { + 'id': '111', + 'name': 'project_1', + 'description': '', + }, + datetime(2017, 5, 17, 19) + ) + + svc = collector.CollectorService() + svc.collect_usage() + + self.assertNotEqual( + [mock.call('111'), mock.call('222'), mock.call('333'), + mock.call('444')], + mock_getlocks.call_args_list + )