Support project order for collection

The project ids returned from Keystone is in ascending order which may
lead to some projects will never get updated if some unexcepted error
happens during collection.

This patch adds a config option to define the prefered collection
order, currently, 'ascending', 'descending' and 'random' are supported.

Change-Id: Ife97e73fe2ae4533ef4baf1f935e46579902719e
This commit is contained in:
Lingxian Kong 2017-07-10 15:01:21 +12:00
parent 07db108fad
commit d906c733a2
3 changed files with 114 additions and 1 deletions

View File

@ -68,7 +68,11 @@ COLLECTOR_OPTS = [
help=('The earlist starting time for new tenant.')),
cfg.StrOpt('partitioning_suffix',
help=('Collector partitioning group suffix. It is used when '
'running multiple collectors in favor of lock.'))
'running multiple collectors in favor of lock.')),
cfg.StrOpt('project_order', default='ascending',
choices=['ascending', 'descending', 'random'],
help=('The order of project IDs to do usage collection. '
'Default is ascending.'))
]
ODOO_OPTS = [

View File

@ -13,6 +13,7 @@
# under the License.
from datetime import datetime
from random import shuffle
from oslo_config import cfg
from oslo_log import log as logging
@ -96,6 +97,16 @@ class CollectorService(service.Service):
super(CollectorService, self).reset()
logging.setup(CONF, 'distil-collector')
def _get_projects_by_order(self, projects):
if CONF.collector.project_order == 'ascending':
return projects
elif CONF.collector.project_order == 'descending':
projects.reverse()
return projects
elif CONF.collector.project_order == 'random':
shuffle(projects)
return projects
def collect_usage(self):
LOG.info("Starting to collect usage...")
@ -110,6 +121,7 @@ class CollectorService(service.Service):
end = datetime.utcnow().replace(minute=0, second=0, microsecond=0)
count = 0
valid_projects = self._get_projects_by_order(valid_projects)
for project in valid_projects:
# Check if the project is being processed by other collector
# instance. If no, will get a lock and continue processing,

View File

@ -154,3 +154,100 @@ class CollectorTest(base.DistilWithDbTestCase):
project_1_collect,
end
)
@mock.patch('distil.db.api.get_project_locks')
@mock.patch('distil.common.openstack.get_ceilometer_client')
@mock.patch('distil.common.openstack.get_projects')
def test_project_order_ascending(self, mock_get_projects, mock_cclient,
mock_getlocks):
mock_get_projects.return_value = [
{'id': '111', 'name': 'project_1', 'description': ''},
{'id': '222', 'name': 'project_2', 'description': ''},
{'id': '333', 'name': 'project_3', 'description': ''},
{'id': '444', 'name': 'project_4', 'description': ''},
]
# Insert a project in the database in order to get last_collect time.
db_api.project_add(
{
'id': '111',
'name': 'project_1',
'description': '',
},
datetime(2017, 5, 17, 19)
)
svc = collector.CollectorService()
svc.collect_usage()
self.assertEqual(
[mock.call('111'), mock.call('222'), mock.call('333'),
mock.call('444')],
mock_getlocks.call_args_list
)
@mock.patch('distil.db.api.get_project_locks')
@mock.patch('distil.common.openstack.get_ceilometer_client')
@mock.patch('distil.common.openstack.get_projects')
def test_project_order_descending(self, mock_get_projects, mock_cclient,
mock_getlocks):
self.override_config('collector', project_order='descending')
mock_get_projects.return_value = [
{'id': '111', 'name': 'project_1', 'description': ''},
{'id': '222', 'name': 'project_2', 'description': ''},
{'id': '333', 'name': 'project_3', 'description': ''},
{'id': '444', 'name': 'project_4', 'description': ''},
]
# Insert a project in the database in order to get last_collect time.
db_api.project_add(
{
'id': '111',
'name': 'project_1',
'description': '',
},
datetime(2017, 5, 17, 19)
)
svc = collector.CollectorService()
svc.collect_usage()
self.assertEqual(
[mock.call('444'), mock.call('333'), mock.call('222'),
mock.call('111')],
mock_getlocks.call_args_list
)
@mock.patch('distil.db.api.get_project_locks')
@mock.patch('distil.common.openstack.get_ceilometer_client')
@mock.patch('distil.common.openstack.get_projects')
def test_project_order_random(self, mock_get_projects, mock_cclient,
mock_getlocks):
self.override_config('collector', project_order='random')
mock_get_projects.return_value = [
{'id': '111', 'name': 'project_1', 'description': ''},
{'id': '222', 'name': 'project_2', 'description': ''},
{'id': '333', 'name': 'project_3', 'description': ''},
{'id': '444', 'name': 'project_4', 'description': ''},
]
# Insert a project in the database in order to get last_collect time.
db_api.project_add(
{
'id': '111',
'name': 'project_1',
'description': '',
},
datetime(2017, 5, 17, 19)
)
svc = collector.CollectorService()
svc.collect_usage()
self.assertNotEqual(
[mock.call('111'), mock.call('222'), mock.call('333'),
mock.call('444')],
mock_getlocks.call_args_list
)