Support project order for collection
The project ids returned from Keystone is in ascending order which may lead to some projects will never get updated if some unexcepted error happens during collection. This patch adds a config option to define the prefered collection order, currently, 'ascending', 'descending' and 'random' are supported. Change-Id: Ife97e73fe2ae4533ef4baf1f935e46579902719e
This commit is contained in:
parent
07db108fad
commit
d906c733a2
|
@ -68,7 +68,11 @@ COLLECTOR_OPTS = [
|
|||
help=('The earlist starting time for new tenant.')),
|
||||
cfg.StrOpt('partitioning_suffix',
|
||||
help=('Collector partitioning group suffix. It is used when '
|
||||
'running multiple collectors in favor of lock.'))
|
||||
'running multiple collectors in favor of lock.')),
|
||||
cfg.StrOpt('project_order', default='ascending',
|
||||
choices=['ascending', 'descending', 'random'],
|
||||
help=('The order of project IDs to do usage collection. '
|
||||
'Default is ascending.'))
|
||||
]
|
||||
|
||||
ODOO_OPTS = [
|
||||
|
|
|
@ -13,6 +13,7 @@
|
|||
# under the License.
|
||||
|
||||
from datetime import datetime
|
||||
from random import shuffle
|
||||
|
||||
from oslo_config import cfg
|
||||
from oslo_log import log as logging
|
||||
|
@ -96,6 +97,16 @@ class CollectorService(service.Service):
|
|||
super(CollectorService, self).reset()
|
||||
logging.setup(CONF, 'distil-collector')
|
||||
|
||||
def _get_projects_by_order(self, projects):
|
||||
if CONF.collector.project_order == 'ascending':
|
||||
return projects
|
||||
elif CONF.collector.project_order == 'descending':
|
||||
projects.reverse()
|
||||
return projects
|
||||
elif CONF.collector.project_order == 'random':
|
||||
shuffle(projects)
|
||||
return projects
|
||||
|
||||
def collect_usage(self):
|
||||
LOG.info("Starting to collect usage...")
|
||||
|
||||
|
@ -110,6 +121,7 @@ class CollectorService(service.Service):
|
|||
end = datetime.utcnow().replace(minute=0, second=0, microsecond=0)
|
||||
count = 0
|
||||
|
||||
valid_projects = self._get_projects_by_order(valid_projects)
|
||||
for project in valid_projects:
|
||||
# Check if the project is being processed by other collector
|
||||
# instance. If no, will get a lock and continue processing,
|
||||
|
|
|
@ -154,3 +154,100 @@ class CollectorTest(base.DistilWithDbTestCase):
|
|||
project_1_collect,
|
||||
end
|
||||
)
|
||||
|
||||
@mock.patch('distil.db.api.get_project_locks')
|
||||
@mock.patch('distil.common.openstack.get_ceilometer_client')
|
||||
@mock.patch('distil.common.openstack.get_projects')
|
||||
def test_project_order_ascending(self, mock_get_projects, mock_cclient,
|
||||
mock_getlocks):
|
||||
mock_get_projects.return_value = [
|
||||
{'id': '111', 'name': 'project_1', 'description': ''},
|
||||
{'id': '222', 'name': 'project_2', 'description': ''},
|
||||
{'id': '333', 'name': 'project_3', 'description': ''},
|
||||
{'id': '444', 'name': 'project_4', 'description': ''},
|
||||
]
|
||||
|
||||
# Insert a project in the database in order to get last_collect time.
|
||||
db_api.project_add(
|
||||
{
|
||||
'id': '111',
|
||||
'name': 'project_1',
|
||||
'description': '',
|
||||
},
|
||||
datetime(2017, 5, 17, 19)
|
||||
)
|
||||
|
||||
svc = collector.CollectorService()
|
||||
svc.collect_usage()
|
||||
|
||||
self.assertEqual(
|
||||
[mock.call('111'), mock.call('222'), mock.call('333'),
|
||||
mock.call('444')],
|
||||
mock_getlocks.call_args_list
|
||||
)
|
||||
|
||||
@mock.patch('distil.db.api.get_project_locks')
|
||||
@mock.patch('distil.common.openstack.get_ceilometer_client')
|
||||
@mock.patch('distil.common.openstack.get_projects')
|
||||
def test_project_order_descending(self, mock_get_projects, mock_cclient,
|
||||
mock_getlocks):
|
||||
self.override_config('collector', project_order='descending')
|
||||
|
||||
mock_get_projects.return_value = [
|
||||
{'id': '111', 'name': 'project_1', 'description': ''},
|
||||
{'id': '222', 'name': 'project_2', 'description': ''},
|
||||
{'id': '333', 'name': 'project_3', 'description': ''},
|
||||
{'id': '444', 'name': 'project_4', 'description': ''},
|
||||
]
|
||||
|
||||
# Insert a project in the database in order to get last_collect time.
|
||||
db_api.project_add(
|
||||
{
|
||||
'id': '111',
|
||||
'name': 'project_1',
|
||||
'description': '',
|
||||
},
|
||||
datetime(2017, 5, 17, 19)
|
||||
)
|
||||
|
||||
svc = collector.CollectorService()
|
||||
svc.collect_usage()
|
||||
|
||||
self.assertEqual(
|
||||
[mock.call('444'), mock.call('333'), mock.call('222'),
|
||||
mock.call('111')],
|
||||
mock_getlocks.call_args_list
|
||||
)
|
||||
|
||||
@mock.patch('distil.db.api.get_project_locks')
|
||||
@mock.patch('distil.common.openstack.get_ceilometer_client')
|
||||
@mock.patch('distil.common.openstack.get_projects')
|
||||
def test_project_order_random(self, mock_get_projects, mock_cclient,
|
||||
mock_getlocks):
|
||||
self.override_config('collector', project_order='random')
|
||||
|
||||
mock_get_projects.return_value = [
|
||||
{'id': '111', 'name': 'project_1', 'description': ''},
|
||||
{'id': '222', 'name': 'project_2', 'description': ''},
|
||||
{'id': '333', 'name': 'project_3', 'description': ''},
|
||||
{'id': '444', 'name': 'project_4', 'description': ''},
|
||||
]
|
||||
|
||||
# Insert a project in the database in order to get last_collect time.
|
||||
db_api.project_add(
|
||||
{
|
||||
'id': '111',
|
||||
'name': 'project_1',
|
||||
'description': '',
|
||||
},
|
||||
datetime(2017, 5, 17, 19)
|
||||
)
|
||||
|
||||
svc = collector.CollectorService()
|
||||
svc.collect_usage()
|
||||
|
||||
self.assertNotEqual(
|
||||
[mock.call('111'), mock.call('222'), mock.call('333'),
|
||||
mock.call('444')],
|
||||
mock_getlocks.call_args_list
|
||||
)
|
||||
|
|
Loading…
Reference in New Issue