summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJenkins <jenkins@review.openstack.org>2017-02-09 12:37:36 +0000
committerGerrit Code Review <review@openstack.org>2017-02-09 12:37:36 +0000
commit15c2ff534b1065f2c4d9f6245c84cc09a59e7a12 (patch)
tree8c2c059d581aacf0d77665da9289882b054ee46e
parent4d025aa0ab1264fab573141abf0750d8c2321065 (diff)
parent88d7bb28b31f6807172969522a2a1dedd80e0c53 (diff)
Merge "Support pagination for OpenStack services"
-rw-r--r--deployment_scripts/puppet/modules/lma_collector/files/collectd/collectd_openstack.py83
-rw-r--r--deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_cinder.py8
-rw-r--r--deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_glance.py5
-rw-r--r--deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_neutron.py1
-rw-r--r--deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_nova.py5
5 files changed, 72 insertions, 30 deletions
diff --git a/deployment_scripts/puppet/modules/lma_collector/files/collectd/collectd_openstack.py b/deployment_scripts/puppet/modules/lma_collector/files/collectd/collectd_openstack.py
index 7108d04..a4ec4de 100644
--- a/deployment_scripts/puppet/modules/lma_collector/files/collectd/collectd_openstack.py
+++ b/deployment_scripts/puppet/modules/lma_collector/files/collectd/collectd_openstack.py
@@ -115,7 +115,8 @@ class OSClient(object):
115 self.logger.debug("Got token '%s'" % self.token) 115 self.logger.debug("Got token '%s'" % self.token)
116 return self.token 116 return self.token
117 117
118 def make_request(self, verb, url, data=None, token_required=True): 118 def make_request(self, verb, url, data=None, token_required=True,
119 params=None):
119 kwargs = { 120 kwargs = {
120 'url': url, 121 'url': url,
121 'timeout': self.timeout, 122 'timeout': self.timeout,
@@ -131,6 +132,9 @@ class OSClient(object):
131 if data is not None: 132 if data is not None:
132 kwargs['data'] = data 133 kwargs['data'] = data
133 134
135 if params is not None:
136 kwargs['params'] = params
137
134 func = getattr(self.session, verb.lower()) 138 func = getattr(self.session, verb.lower())
135 139
136 try: 140 try:
@@ -159,6 +163,7 @@ class CollectdPlugin(base.Base):
159 self.max_retries = 2 163 self.max_retries = 2
160 self.os_client = None 164 self.os_client = None
161 self.extra_config = {} 165 self.extra_config = {}
166 self.pagination_limit = None
162 167
163 def _build_url(self, service, resource): 168 def _build_url(self, service, resource):
164 s = (self.get_service(service) or {}) 169 s = (self.get_service(service) or {})
@@ -241,12 +246,12 @@ class CollectdPlugin(base.Base):
241 246
242 yield data 247 yield data
243 248
244 def get(self, service, resource): 249 def get(self, service, resource, params=None):
245 url = self._build_url(service, resource) 250 url = self._build_url(service, resource)
246 if not url: 251 if not url:
247 return 252 return
248 self.logger.info("GET '%s'" % url) 253 self.logger.info("GET '%s'" % url)
249 return self.os_client.make_request('get', url) 254 return self.os_client.make_request('get', url, params=params)
250 255
251 @property 256 @property
252 def service_catalog(self): 257 def service_catalog(self):
@@ -271,46 +276,72 @@ class CollectdPlugin(base.Base):
271 tenant_name = node.values[0] 276 tenant_name = node.values[0]
272 elif node.key == 'KeystoneUrl': 277 elif node.key == 'KeystoneUrl':
273 keystone_url = node.values[0] 278 keystone_url = node.values[0]
279 elif node.key == 'PaginationLimit':
280 self.pagination_limit = int(node.values[0])
274 self.os_client = OSClient(username, password, tenant_name, 281 self.os_client = OSClient(username, password, tenant_name,
275 keystone_url, self.timeout, self.logger, 282 keystone_url, self.timeout, self.logger,
276 self.max_retries) 283 self.max_retries)
277 284
278 def get_objects(self, project, object_name, api_version='', 285 def get_objects(self, project, object_name, api_version='',
279 params='all_tenants=1'): 286 params=None, detail=False):
280 """ Return a list of OpenStack objects 287 """ Return a list of OpenStack objects
281 288
282 See get_objects_details()
283 """
284 return self._get_objects(project, object_name, api_version, params,
285 False)
286
287 def get_objects_details(self, project, object_name, api_version='',
288 params='all_tenants=1'):
289 """ Return a list of details about OpenStack objects
290
291 The API version is not always included in the URL endpoint 289 The API version is not always included in the URL endpoint
292 registered in Keystone (eg Glance). In this case, use the 290 registered in Keystone (eg Glance). In this case, use the
293 api_version parameter to specify which version should be used. 291 api_version parameter to specify which version should be used.
292
294 """ 293 """
295 return self._get_objects(project, object_name, api_version, params, 294 if params is None:
296 True) 295 params = {}
297 296
298 def _get_objects(self, project, object_name, api_version, params, detail):
299 if api_version: 297 if api_version:
300 resource = '%s/%s' % (api_version, object_name) 298 resource = '%s/%s' % (api_version, object_name)
301 else: 299 else:
302 resource = '%s' % (object_name) 300 resource = '%s' % (object_name)
301
303 if detail: 302 if detail:
304 resource = '%s/detail' % (resource) 303 resource = '{}/detail'.format(resource)
305 if params: 304
306 resource = '%s?%s' % (resource, params) 305 url = self._build_url(project, resource)
307 # TODO(scroiset): use pagination to handle large collection 306 if not url:
308 r = self.get(project, resource) 307 return
309 if not r or object_name not in r.json(): 308
310 self.logger.warning('Could not find %s %s' % (project, 309 opts = {}
311 object_name)) 310 if self.pagination_limit:
312 return [] 311 opts['limit'] = self.pagination_limit
313 return r.json().get(object_name) 312
313 opts.update(params)
314 objs = []
315
316 while True:
317 r = self.os_client.make_request('get', url, params=opts)
318 if not r or object_name not in r.json():
319 self.logger.warning('Could not find %s %s' % (project,
320 object_name))
321 return objs
322
323 resp = r.json()
324 bulk_objs = resp.get(object_name)
325
326 if not bulk_objs:
327 break
328
329 objs.extend(bulk_objs)
330
331 links = resp.get('{}_links'.format(object_name))
332 if links is None or self.pagination_limit is None:
333 # Either the pagination is not supported or there is no more
334 # data
335 break
336
337 # if there is no 'next' link in the response, all data has been
338 # read.
339 if len([i for i in links if i.get('rel') == 'next']) == 0:
340 break
341
342 opts['marker'] = bulk_objs[-1]['id']
343
344 return objs
314 345
315 def count_objects_group_by(self, 346 def count_objects_group_by(self,
316 list_object, 347 list_object,
diff --git a/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_cinder.py b/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_cinder.py
index 373c081..8211f71 100644
--- a/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_cinder.py
+++ b/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_cinder.py
@@ -33,10 +33,13 @@ class CinderStatsPlugin(openstack.CollectdPlugin):
33 super(CinderStatsPlugin, self).__init__(*args, **kwargs) 33 super(CinderStatsPlugin, self).__init__(*args, **kwargs)
34 self.plugin = PLUGIN_NAME 34 self.plugin = PLUGIN_NAME
35 self.interval = INTERVAL 35 self.interval = INTERVAL
36 self.pagination_limit = 500
36 37
37 def itermetrics(self): 38 def itermetrics(self):
38 39
39 volumes_details = self.get_objects_details('cinder', 'volumes') 40 volumes_details = self.get_objects('cinder', 'volumes',
41 params={'all_tenants': 1},
42 detail=True)
40 43
41 def groupby(d): 44 def groupby(d):
42 return d.get('status', 'unknown').lower() 45 return d.get('status', 'unknown').lower()
@@ -63,7 +66,8 @@ class CinderStatsPlugin(openstack.CollectdPlugin):
63 'values': size 66 'values': size
64 } 67 }
65 68
66 snaps_details = self.get_objects_details('cinder', 'snapshots') 69 snaps_details = self.get_objects('cinder', 'snapshots',
70 params={'all_tenants': 1})
67 status_snaps = self.count_objects_group_by(snaps_details, 71 status_snaps = self.count_objects_group_by(snaps_details,
68 group_by_func=groupby) 72 group_by_func=groupby)
69 for s, nb in status_snaps.iteritems(): 73 for s, nb in status_snaps.iteritems():
diff --git a/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_glance.py b/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_glance.py
index a6b451f..bcb9712 100644
--- a/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_glance.py
+++ b/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_glance.py
@@ -33,6 +33,7 @@ class GlanceStatsPlugin(openstack.CollectdPlugin):
33 super(GlanceStatsPlugin, self).__init__(*args, **kwargs) 33 super(GlanceStatsPlugin, self).__init__(*args, **kwargs)
34 self.plugin = PLUGIN_NAME 34 self.plugin = PLUGIN_NAME
35 self.interval = INTERVAL 35 self.interval = INTERVAL
36 self.pagination_limit = 25
36 37
37 def itermetrics(self): 38 def itermetrics(self):
38 39
@@ -48,7 +49,9 @@ class GlanceStatsPlugin(openstack.CollectdPlugin):
48 49
49 images_details = self.get_objects_details('glance', 'images', 50 images_details = self.get_objects_details('glance', 'images',
50 api_version='v1', 51 api_version='v1',
51 params='is_public=None') 52 params={},
53 detail=True)
54
52 status = self.count_objects_group_by(images_details, 55 status = self.count_objects_group_by(images_details,
53 group_by_func=groupby) 56 group_by_func=groupby)
54 for s, nb in status.iteritems(): 57 for s, nb in status.iteritems():
diff --git a/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_neutron.py b/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_neutron.py
index 798f091..e677ec1 100644
--- a/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_neutron.py
+++ b/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_neutron.py
@@ -36,6 +36,7 @@ class NeutronStatsPlugin(openstack.CollectdPlugin):
36 super(NeutronStatsPlugin, self).__init__(*args, **kwargs) 36 super(NeutronStatsPlugin, self).__init__(*args, **kwargs)
37 self.plugin = PLUGIN_NAME 37 self.plugin = PLUGIN_NAME
38 self.interval = INTERVAL 38 self.interval = INTERVAL
39 self.pagination_limit = 100
39 40
40 def itermetrics(self): 41 def itermetrics(self):
41 42
diff --git a/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_nova.py b/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_nova.py
index 455e5d1..8cdd57d 100644
--- a/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_nova.py
+++ b/deployment_scripts/puppet/modules/lma_collector/files/collectd/openstack_nova.py
@@ -31,9 +31,12 @@ class NovaInstanceStatsPlugin(openstack.CollectdPlugin):
31 super(NovaInstanceStatsPlugin, self).__init__(*args, **kwargs) 31 super(NovaInstanceStatsPlugin, self).__init__(*args, **kwargs)
32 self.plugin = PLUGIN_NAME 32 self.plugin = PLUGIN_NAME
33 self.interval = INTERVAL 33 self.interval = INTERVAL
34 self.pagination_limit = 500
34 35
35 def itermetrics(self): 36 def itermetrics(self):
36 servers_details = self.get_objects_details('nova', 'servers') 37 servers_details = self.get_objects('nova', 'servers',
38 params={'all_tenants': 1},
39 detail=True)
37 40
38 def groupby(d): 41 def groupby(d):
39 return d.get('status', 'unknown').lower() 42 return d.get('status', 'unknown').lower()