diff --git a/manila/share/driver.py b/manila/share/driver.py index 465ae91bc4..5ddb6eec3c 100644 --- a/manila/share/driver.py +++ b/manila/share/driver.py @@ -2422,3 +2422,27 @@ class ShareDriver(object): :param share_server: None or Share server model """ raise NotImplementedError() + + def update_share_usage_size(self, context, shares): + """Invoked to get the usage size of given shares. + + Driver can use this method to update the share usage size of + the shares. To do that, a dictionary of shares should be + returned. + :param shares: None or a list of all shares for updates. + :returns: An empty list or a list of dictionary of updates in the + following format. The value of "used_size" can be specified in GiB + units, as a floating point number:: + + [ + { + 'id': '09960614-8574-4e03-89cf-7cf267b0bd08', + 'used_size': '200', + 'gathered_at': datetime.datetime(2017, 8, 10, 15, 14, 6), + }, + ] + + """ + LOG.debug("This backend does not support gathering 'used_size' of " + "shares created on it.") + return [] diff --git a/manila/share/drivers/lvm.py b/manila/share/drivers/lvm.py index 598952f411..2ac191104f 100644 --- a/manila/share/drivers/lvm.py +++ b/manila/share/drivers/lvm.py @@ -25,6 +25,7 @@ import re from oslo_config import cfg from oslo_log import log from oslo_utils import importutils +from oslo_utils import timeutils import six from manila import exception @@ -444,3 +445,30 @@ class LVMShareDriver(LVMMixin, driver.ShareDriver): helper.update_access(self.share_server, snapshot['name'], access_rules, add_rules=add_rules, delete_rules=delete_rules) + + def update_share_usage_size(self, context, shares): + updated_shares = [] + out, err = self._execute( + 'df', '-l', '--output=target,used', + '--block-size=g') + gathered_at = timeutils.utcnow() + + for share in shares: + try: + mount_path = self._get_mount_path(share) + if os.path.exists(mount_path): + used_size = (re.findall( + mount_path + "\s*[0-9.]+G", out)[0]. + split(' ')[-1][:-1]) + updated_shares.append({'id': share['id'], + 'used_size': used_size, + 'gathered_at': gathered_at}) + else: + raise exception.NotFound( + _("Share mount path %s could not be " + "found.") % mount_path) + except Exception: + LOG.exception("Failed to gather 'used_size' for share %s.", + share['id']) + + return updated_shares diff --git a/manila/share/manager.py b/manila/share/manager.py index ec6e47bcd4..c019f699ba 100644 --- a/manila/share/manager.py +++ b/manila/share/manager.py @@ -104,6 +104,21 @@ share_manager_opts = [ 'the share manager will poll the driver to perform the ' 'next step of migration in the storage backend, for a ' 'migrating share.'), + cfg.IntOpt('share_usage_size_update_interval', + default=300, + help='This value, specified in seconds, determines how often ' + 'the share manager will poll the driver to update the ' + 'share usage size in the storage backend, for shares in ' + 'that backend.'), + cfg.BoolOpt('enable_gathering_share_usage_size', + default=False, + help='If set to True, share usage size will be polled for in ' + 'the interval specified with ' + '"share_usage_size_update_interval". Usage data can be ' + 'consumed by telemetry integration. If telemetry is not ' + 'configured, this option must be set to False. ' + 'If set to False - gathering share usage size will be' + ' disabled.'), ] CONF = cfg.CONF @@ -3849,3 +3864,28 @@ class ShareManager(manager.SchedulerDependentManager): share_utils.notify_about_share_usage( context, share, share_instance, event_suffix, extra_usage_info=extra_usage_info, host=self.host) + + @periodic_task.periodic_task( + spacing=CONF.share_usage_size_update_interval, + enabled=CONF.enable_gathering_share_usage_size) + @utils.require_driver_initialized + def update_share_usage_size(self, context): + """Invokes driver to gather usage size of shares.""" + updated_share_instances = [] + share_instances = self.db.share_instances_get_all_by_host( + context, host=self.host, with_share_data=True) + + if share_instances: + try: + updated_share_instances = self.driver.update_share_usage_size( + context, share_instances) + except Exception: + LOG.exception("Gather share usage size failure.") + + for si in updated_share_instances: + share_instance = self._get_share_instance(context, si['id']) + share = self.db.share_get(context, share_instance['share_id']) + self._notify_about_share_usage( + context, share, share_instance, "consumed.size", + extra_usage_info={'used_size': si['used_size'], + 'gathered_at': si['gathered_at']}) diff --git a/manila/tests/share/drivers/dummy.py b/manila/tests/share/drivers/dummy.py index 1722dad7e5..cfdb1ad161 100644 --- a/manila/tests/share/drivers/dummy.py +++ b/manila/tests/share/drivers/dummy.py @@ -34,6 +34,7 @@ import time from oslo_config import cfg from oslo_log import log +from oslo_utils import timeutils from manila.common import constants from manila import exception @@ -628,3 +629,12 @@ class DummyDriver(driver.ShareDriver): "progress": total_progress }) return {"total_progress": total_progress} + + def update_share_usage_size(self, context, shares): + share_updates = [] + gathered_at = timeutils.utcnow() + for s in shares: + share_updates.append({'id': s['id'], + 'used_size': 1, + 'gathered_at': gathered_at}) + return share_updates diff --git a/manila/tests/share/drivers/test_lvm.py b/manila/tests/share/drivers/test_lvm.py index 1acdad3857..ef8fa00ce5 100644 --- a/manila/tests/share/drivers/test_lvm.py +++ b/manila/tests/share/drivers/test_lvm.py @@ -19,6 +19,7 @@ import os import ddt import mock from oslo_config import cfg +from oslo_utils import timeutils from manila.common import constants as const from manila import context @@ -591,3 +592,64 @@ class LVMShareDriverTestCase(test.TestCase): update_access.assert_called_once_with( self.server, self.snapshot['name'], access_rules, add_rules=add_rules, delete_rules=delete_rules)) + + @mock.patch.object(timeutils, 'utcnow', mock.Mock( + return_value='fake_date')) + def test_update_share_usage_size(self): + mount_path = self._get_mount_path(self.share) + self._os.path.exists.return_value = True + self.mock_object( + self._driver, + '_execute', + mock.Mock(return_value=( + "Mounted on Used " + + mount_path + " 1G", None))) + + update_shares = self._driver.update_share_usage_size( + self._context, [self.share, ]) + self._os.path.exists.assert_called_with(mount_path) + self.assertEqual( + [{'id': 'fakeid', 'used_size': '1', + 'gathered_at': 'fake_date'}], + update_shares) + self._driver._execute.assert_called_once_with( + 'df', '-l', '--output=target,used', + '--block-size=g') + + @mock.patch.object(timeutils, 'utcnow', mock.Mock( + return_value='fake_date')) + def test_update_share_usage_size_multiple_share(self): + share1 = fake_share(id='fakeid_get_fail', name='get_fail') + share2 = fake_share(id='fakeid_success', name='get_success') + share3 = fake_share(id='fakeid_not_exist', name='get_not_exist') + + mount_path2 = self._get_mount_path(share2) + mount_path3 = self._get_mount_path(share3) + self._os.path.exists.side_effect = [True, True, False] + self.mock_object( + self._driver, + '_execute', + mock.Mock(return_value=( + "Mounted on Used " + + mount_path2 + " 1G", None))) + + update_shares = self._driver.update_share_usage_size( + self._context, [share1, share2, share3]) + self._os.path.exists.assert_called_with(mount_path3) + self.assertEqual( + [{'gathered_at': 'fake_date', + 'id': 'fakeid_success', 'used_size': '1'}], + update_shares) + self._driver._execute.assert_called_with( + 'df', '-l', '--output=target,used', + '--block-size=g') + + def test_update_share_usage_size_fail(self): + def _fake_exec(*args, **kwargs): + raise exception.ProcessExecutionError(stderr="error") + + self.mock_object(self._driver, '_execute', _fake_exec) + self.assertRaises(exception.ProcessExecutionError, + self._driver.update_share_usage_size, + self._context, + [self.share]) diff --git a/manila/tests/share/test_manager.py b/manila/tests/share/test_manager.py index 1ade84b04c..3fcfdac5e5 100644 --- a/manila/tests/share/test_manager.py +++ b/manila/tests/share/test_manager.py @@ -6303,6 +6303,46 @@ class ShareManagerTestCase(test.TestCase): self.context, share_instance['id'], share_server='fake_share_server') + @mock.patch('manila.tests.fake_notifier.FakeNotifier._notify') + def test_update_share_usage_size(self, mock_notify): + instances = self._setup_init_mocks(setup_access_rules=False) + update_shares = [{'id': 'fake_id', 'used_size': '3', + 'gathered_at': 'fake'}] + mock_notify.assert_not_called() + + manager = self.share_manager + self.mock_object(manager, 'driver') + self.mock_object(manager.db, 'share_instances_get_all_by_host', + mock.Mock(return_value=instances)) + self.mock_object(manager.db, 'share_instance_get', + mock.Mock(side_effect=instances)) + mock_driver_call = self.mock_object( + manager.driver, 'update_share_usage_size', + mock.Mock(return_value=update_shares)) + self.share_manager.update_share_usage_size(self.context) + self.assert_notify_called(mock_notify, + (['INFO', 'share.consumed.size'], )) + mock_driver_call.assert_called_once_with( + self.context, instances) + + @mock.patch('manila.tests.fake_notifier.FakeNotifier._notify') + def test_update_share_usage_size_fail(self, mock_notify): + instances = self._setup_init_mocks(setup_access_rules=False) + mock_notify.assert_not_called() + + self.mock_object(self.share_manager, 'driver') + self.mock_object(self.share_manager.db, + 'share_instances_get_all_by_host', + mock.Mock(return_value=instances)) + self.mock_object(self.share_manager.db, 'share_instance_get', + mock.Mock(side_effect=instances)) + self.mock_object( + self.share_manager.driver, 'update_share_usage_size', + mock.Mock(side_effect=exception.ProcessExecutionError)) + mock_log_exception = self.mock_object(manager.LOG, 'exception') + self.share_manager.update_share_usage_size(self.context) + self.assertTrue(mock_log_exception.called) + @ddt.ddt class HookWrapperTestCase(test.TestCase): diff --git a/releasenotes/notes/add-gathering-usage-size-8454sd45deopb14e.yaml b/releasenotes/notes/add-gathering-usage-size-8454sd45deopb14e.yaml new file mode 100644 index 0000000000..09838dcf7d --- /dev/null +++ b/releasenotes/notes/add-gathering-usage-size-8454sd45deopb14e.yaml @@ -0,0 +1,9 @@ +--- +features: + - Added periodic task to gather share usage size. +upgrade: + - Added enable_gathering_share_usage_size and + share_usage_size_update_interval options in the + manila.conf file to allow configuration of gathering + share usage size support and to allow configuration + of interval time of gathering share usage size.