openstack
/
manila
Code Issues Proposed changes

Add share usage size tracking 

We need to have a way to gather information about actual share storage
usages, so cloud operators could use this information for billing,
health checks and/or other purposes.

Change-Id: Iaca1bb541a34af862b938e17e4a56d53de7a9cc4
Implement-Blueprint: share-usage-size
This commit is contained in:
zhongjun 2017-06-30 18:18:25 +08:00 committed by zhongjun
parent 5bb153399b
commit f68c1a4080
7 changed files with 213 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
manila/share/driver.py
View File

@ -2422,3 +2422,27 @@ class ShareDriver(object):
:param share_server: None or Share server model
"""
raise NotImplementedError()
def update_share_usage_size(self, context, shares):
"""Invoked to get the usage size of given shares.
Driver can use this method to update the share usage size of
the shares. To do that, a dictionary of shares should be
returned.
:param shares: None or a list of all shares for updates.
:returns: An empty list or a list of dictionary of updates in the
following format. The value of "used_size" can be specified in GiB
units, as a floating point number::
[
{
'id': '09960614-8574-4e03-89cf-7cf267b0bd08',
'used_size': '200',
'gathered_at': datetime.datetime(2017, 8, 10, 15, 14, 6),
},
]
"""
LOG.debug("This backend does not support gathering 'used_size' of "
"shares created on it.")
return []

28
manila/share/drivers/lvm.py
View File

@ -25,6 +25,7 @@ import re
from oslo_config import cfg
from oslo_log import log
from oslo_utils import importutils
from oslo_utils import timeutils
import six
from manila import exception
@ -444,3 +445,30 @@ class LVMShareDriver(LVMMixin, driver.ShareDriver):
helper.update_access(self.share_server,
snapshot['name'], access_rules,
add_rules=add_rules, delete_rules=delete_rules)
def update_share_usage_size(self, context, shares):
updated_shares = []
out, err = self._execute(
'df', '-l', '--output=target,used',
'--block-size=g')
gathered_at = timeutils.utcnow()
for share in shares:
try:
mount_path = self._get_mount_path(share)
if os.path.exists(mount_path):
used_size = (re.findall(
mount_path + "\s*[0-9.]+G", out)[0].
split(' ')[-1][:-1])
updated_shares.append({'id': share['id'],
'used_size': used_size,
'gathered_at': gathered_at})
else:
raise exception.NotFound(
_("Share mount path %s could not be "
"found.") % mount_path)
except Exception:
LOG.exception("Failed to gather 'used_size' for share %s.",
share['id'])
return updated_shares

40
manila/share/manager.py
View File

@ -104,6 +104,21 @@ share_manager_opts = [
'the share manager will poll the driver to perform the '
'next step of migration in the storage backend, for a '
'migrating share.'),
cfg.IntOpt('share_usage_size_update_interval',
default=300,
help='This value, specified in seconds, determines how often '
'the share manager will poll the driver to update the '
'share usage size in the storage backend, for shares in '
'that backend.'),
cfg.BoolOpt('enable_gathering_share_usage_size',
default=False,
help='If set to True, share usage size will be polled for in '
'the interval specified with '
'"share_usage_size_update_interval". Usage data can be '
'consumed by telemetry integration. If telemetry is not '
'configured, this option must be set to False. '
'If set to False - gathering share usage size will be'
' disabled.'),
]
CONF = cfg.CONF
@ -3849,3 +3864,28 @@ class ShareManager(manager.SchedulerDependentManager):
share_utils.notify_about_share_usage(
context, share, share_instance, event_suffix,
extra_usage_info=extra_usage_info, host=self.host)
@periodic_task.periodic_task(
spacing=CONF.share_usage_size_update_interval,
enabled=CONF.enable_gathering_share_usage_size)
@utils.require_driver_initialized
def update_share_usage_size(self, context):
"""Invokes driver to gather usage size of shares."""
updated_share_instances = []
share_instances = self.db.share_instances_get_all_by_host(
context, host=self.host, with_share_data=True)
if share_instances:
try:
updated_share_instances = self.driver.update_share_usage_size(
context, share_instances)
except Exception:
LOG.exception("Gather share usage size failure.")
for si in updated_share_instances:
share_instance = self._get_share_instance(context, si['id'])
share = self.db.share_get(context, share_instance['share_id'])
self._notify_about_share_usage(
context, share, share_instance, "consumed.size",
extra_usage_info={'used_size': si['used_size'],
'gathered_at': si['gathered_at']})

10
manila/tests/share/drivers/dummy.py
View File

@ -34,6 +34,7 @@ import time
from oslo_config import cfg
from oslo_log import log
from oslo_utils import timeutils
from manila.common import constants
from manila import exception
@ -628,3 +629,12 @@ class DummyDriver(driver.ShareDriver):
"progress": total_progress
})
return {"total_progress": total_progress}
def update_share_usage_size(self, context, shares):
share_updates = []
gathered_at = timeutils.utcnow()
for s in shares:
share_updates.append({'id': s['id'],
'used_size': 1,
'gathered_at': gathered_at})
return share_updates

62
manila/tests/share/drivers/test_lvm.py
View File

@ -19,6 +19,7 @@ import os
import ddt
import mock
from oslo_config import cfg
from oslo_utils import timeutils
from manila.common import constants as const
from manila import context
@ -591,3 +592,64 @@ class LVMShareDriverTestCase(test.TestCase):
update_access.assert_called_once_with(
self.server, self.snapshot['name'],
access_rules, add_rules=add_rules, delete_rules=delete_rules))
@mock.patch.object(timeutils, 'utcnow', mock.Mock(
return_value='fake_date'))
def test_update_share_usage_size(self):
mount_path = self._get_mount_path(self.share)
self._os.path.exists.return_value = True
self.mock_object(
self._driver,
'_execute',
mock.Mock(return_value=(
"Mounted on Used "
+ mount_path + " 1G", None)))
update_shares = self._driver.update_share_usage_size(
self._context, [self.share, ])
self._os.path.exists.assert_called_with(mount_path)
self.assertEqual(
[{'id': 'fakeid', 'used_size': '1',
'gathered_at': 'fake_date'}],
update_shares)
self._driver._execute.assert_called_once_with(
'df', '-l', '--output=target,used',
'--block-size=g')
@mock.patch.object(timeutils, 'utcnow', mock.Mock(
return_value='fake_date'))
def test_update_share_usage_size_multiple_share(self):
share1 = fake_share(id='fakeid_get_fail', name='get_fail')
share2 = fake_share(id='fakeid_success', name='get_success')
share3 = fake_share(id='fakeid_not_exist', name='get_not_exist')
mount_path2 = self._get_mount_path(share2)
mount_path3 = self._get_mount_path(share3)
self._os.path.exists.side_effect = [True, True, False]
self.mock_object(
self._driver,
'_execute',
mock.Mock(return_value=(
"Mounted on Used "
+ mount_path2 + " 1G", None)))
update_shares = self._driver.update_share_usage_size(
self._context, [share1, share2, share3])
self._os.path.exists.assert_called_with(mount_path3)
self.assertEqual(
[{'gathered_at': 'fake_date',
'id': 'fakeid_success', 'used_size': '1'}],
update_shares)
self._driver._execute.assert_called_with(
'df', '-l', '--output=target,used',
'--block-size=g')
def test_update_share_usage_size_fail(self):
def _fake_exec(*args, **kwargs):
raise exception.ProcessExecutionError(stderr="error")
self.mock_object(self._driver, '_execute', _fake_exec)
self.assertRaises(exception.ProcessExecutionError,
self._driver.update_share_usage_size,
self._context,
[self.share])

40
manila/tests/share/test_manager.py
View File

@ -6303,6 +6303,46 @@ class ShareManagerTestCase(test.TestCase):
self.context, share_instance['id'],
share_server='fake_share_server')
@mock.patch('manila.tests.fake_notifier.FakeNotifier._notify')
def test_update_share_usage_size(self, mock_notify):
instances = self._setup_init_mocks(setup_access_rules=False)
update_shares = [{'id': 'fake_id', 'used_size': '3',
'gathered_at': 'fake'}]
mock_notify.assert_not_called()
manager = self.share_manager
self.mock_object(manager, 'driver')
self.mock_object(manager.db, 'share_instances_get_all_by_host',
mock.Mock(return_value=instances))
self.mock_object(manager.db, 'share_instance_get',
mock.Mock(side_effect=instances))
mock_driver_call = self.mock_object(
manager.driver, 'update_share_usage_size',
mock.Mock(return_value=update_shares))
self.share_manager.update_share_usage_size(self.context)
self.assert_notify_called(mock_notify,
(['INFO', 'share.consumed.size'], ))
mock_driver_call.assert_called_once_with(
self.context, instances)
@mock.patch('manila.tests.fake_notifier.FakeNotifier._notify')
def test_update_share_usage_size_fail(self, mock_notify):
instances = self._setup_init_mocks(setup_access_rules=False)
mock_notify.assert_not_called()
self.mock_object(self.share_manager, 'driver')
self.mock_object(self.share_manager.db,
'share_instances_get_all_by_host',
mock.Mock(return_value=instances))
self.mock_object(self.share_manager.db, 'share_instance_get',
mock.Mock(side_effect=instances))
self.mock_object(
self.share_manager.driver, 'update_share_usage_size',
mock.Mock(side_effect=exception.ProcessExecutionError))
mock_log_exception = self.mock_object(manager.LOG, 'exception')
self.share_manager.update_share_usage_size(self.context)
self.assertTrue(mock_log_exception.called)
@ddt.ddt
class HookWrapperTestCase(test.TestCase):

9
releasenotes/notes/add-gathering-usage-size-8454sd45deopb14e.yaml Normal file
View File

@ -0,0 +1,9 @@
---
features:
- Added periodic task to gather share usage size.
upgrade:
- Added enable_gathering_share_usage_size and
share_usage_size_update_interval options in the
manila.conf file to allow configuration of gathering
share usage size support and to allow configuration
of interval time of gathering share usage size.