Add share usage size tracking
We need to have a way to gather information about actual share storage usages, so cloud operators could use this information for billing, health checks and/or other purposes. Change-Id: Iaca1bb541a34af862b938e17e4a56d53de7a9cc4 Implement-Blueprint: share-usage-size
This commit is contained in:
parent
5bb153399b
commit
f68c1a4080
|
@ -2422,3 +2422,27 @@ class ShareDriver(object):
|
||||||
:param share_server: None or Share server model
|
:param share_server: None or Share server model
|
||||||
"""
|
"""
|
||||||
raise NotImplementedError()
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
def update_share_usage_size(self, context, shares):
|
||||||
|
"""Invoked to get the usage size of given shares.
|
||||||
|
|
||||||
|
Driver can use this method to update the share usage size of
|
||||||
|
the shares. To do that, a dictionary of shares should be
|
||||||
|
returned.
|
||||||
|
:param shares: None or a list of all shares for updates.
|
||||||
|
:returns: An empty list or a list of dictionary of updates in the
|
||||||
|
following format. The value of "used_size" can be specified in GiB
|
||||||
|
units, as a floating point number::
|
||||||
|
|
||||||
|
[
|
||||||
|
{
|
||||||
|
'id': '09960614-8574-4e03-89cf-7cf267b0bd08',
|
||||||
|
'used_size': '200',
|
||||||
|
'gathered_at': datetime.datetime(2017, 8, 10, 15, 14, 6),
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
"""
|
||||||
|
LOG.debug("This backend does not support gathering 'used_size' of "
|
||||||
|
"shares created on it.")
|
||||||
|
return []
|
||||||
|
|
|
@ -25,6 +25,7 @@ import re
|
||||||
from oslo_config import cfg
|
from oslo_config import cfg
|
||||||
from oslo_log import log
|
from oslo_log import log
|
||||||
from oslo_utils import importutils
|
from oslo_utils import importutils
|
||||||
|
from oslo_utils import timeutils
|
||||||
import six
|
import six
|
||||||
|
|
||||||
from manila import exception
|
from manila import exception
|
||||||
|
@ -444,3 +445,30 @@ class LVMShareDriver(LVMMixin, driver.ShareDriver):
|
||||||
helper.update_access(self.share_server,
|
helper.update_access(self.share_server,
|
||||||
snapshot['name'], access_rules,
|
snapshot['name'], access_rules,
|
||||||
add_rules=add_rules, delete_rules=delete_rules)
|
add_rules=add_rules, delete_rules=delete_rules)
|
||||||
|
|
||||||
|
def update_share_usage_size(self, context, shares):
|
||||||
|
updated_shares = []
|
||||||
|
out, err = self._execute(
|
||||||
|
'df', '-l', '--output=target,used',
|
||||||
|
'--block-size=g')
|
||||||
|
gathered_at = timeutils.utcnow()
|
||||||
|
|
||||||
|
for share in shares:
|
||||||
|
try:
|
||||||
|
mount_path = self._get_mount_path(share)
|
||||||
|
if os.path.exists(mount_path):
|
||||||
|
used_size = (re.findall(
|
||||||
|
mount_path + "\s*[0-9.]+G", out)[0].
|
||||||
|
split(' ')[-1][:-1])
|
||||||
|
updated_shares.append({'id': share['id'],
|
||||||
|
'used_size': used_size,
|
||||||
|
'gathered_at': gathered_at})
|
||||||
|
else:
|
||||||
|
raise exception.NotFound(
|
||||||
|
_("Share mount path %s could not be "
|
||||||
|
"found.") % mount_path)
|
||||||
|
except Exception:
|
||||||
|
LOG.exception("Failed to gather 'used_size' for share %s.",
|
||||||
|
share['id'])
|
||||||
|
|
||||||
|
return updated_shares
|
||||||
|
|
|
@ -104,6 +104,21 @@ share_manager_opts = [
|
||||||
'the share manager will poll the driver to perform the '
|
'the share manager will poll the driver to perform the '
|
||||||
'next step of migration in the storage backend, for a '
|
'next step of migration in the storage backend, for a '
|
||||||
'migrating share.'),
|
'migrating share.'),
|
||||||
|
cfg.IntOpt('share_usage_size_update_interval',
|
||||||
|
default=300,
|
||||||
|
help='This value, specified in seconds, determines how often '
|
||||||
|
'the share manager will poll the driver to update the '
|
||||||
|
'share usage size in the storage backend, for shares in '
|
||||||
|
'that backend.'),
|
||||||
|
cfg.BoolOpt('enable_gathering_share_usage_size',
|
||||||
|
default=False,
|
||||||
|
help='If set to True, share usage size will be polled for in '
|
||||||
|
'the interval specified with '
|
||||||
|
'"share_usage_size_update_interval". Usage data can be '
|
||||||
|
'consumed by telemetry integration. If telemetry is not '
|
||||||
|
'configured, this option must be set to False. '
|
||||||
|
'If set to False - gathering share usage size will be'
|
||||||
|
' disabled.'),
|
||||||
]
|
]
|
||||||
|
|
||||||
CONF = cfg.CONF
|
CONF = cfg.CONF
|
||||||
|
@ -3849,3 +3864,28 @@ class ShareManager(manager.SchedulerDependentManager):
|
||||||
share_utils.notify_about_share_usage(
|
share_utils.notify_about_share_usage(
|
||||||
context, share, share_instance, event_suffix,
|
context, share, share_instance, event_suffix,
|
||||||
extra_usage_info=extra_usage_info, host=self.host)
|
extra_usage_info=extra_usage_info, host=self.host)
|
||||||
|
|
||||||
|
@periodic_task.periodic_task(
|
||||||
|
spacing=CONF.share_usage_size_update_interval,
|
||||||
|
enabled=CONF.enable_gathering_share_usage_size)
|
||||||
|
@utils.require_driver_initialized
|
||||||
|
def update_share_usage_size(self, context):
|
||||||
|
"""Invokes driver to gather usage size of shares."""
|
||||||
|
updated_share_instances = []
|
||||||
|
share_instances = self.db.share_instances_get_all_by_host(
|
||||||
|
context, host=self.host, with_share_data=True)
|
||||||
|
|
||||||
|
if share_instances:
|
||||||
|
try:
|
||||||
|
updated_share_instances = self.driver.update_share_usage_size(
|
||||||
|
context, share_instances)
|
||||||
|
except Exception:
|
||||||
|
LOG.exception("Gather share usage size failure.")
|
||||||
|
|
||||||
|
for si in updated_share_instances:
|
||||||
|
share_instance = self._get_share_instance(context, si['id'])
|
||||||
|
share = self.db.share_get(context, share_instance['share_id'])
|
||||||
|
self._notify_about_share_usage(
|
||||||
|
context, share, share_instance, "consumed.size",
|
||||||
|
extra_usage_info={'used_size': si['used_size'],
|
||||||
|
'gathered_at': si['gathered_at']})
|
||||||
|
|
|
@ -34,6 +34,7 @@ import time
|
||||||
|
|
||||||
from oslo_config import cfg
|
from oslo_config import cfg
|
||||||
from oslo_log import log
|
from oslo_log import log
|
||||||
|
from oslo_utils import timeutils
|
||||||
|
|
||||||
from manila.common import constants
|
from manila.common import constants
|
||||||
from manila import exception
|
from manila import exception
|
||||||
|
@ -628,3 +629,12 @@ class DummyDriver(driver.ShareDriver):
|
||||||
"progress": total_progress
|
"progress": total_progress
|
||||||
})
|
})
|
||||||
return {"total_progress": total_progress}
|
return {"total_progress": total_progress}
|
||||||
|
|
||||||
|
def update_share_usage_size(self, context, shares):
|
||||||
|
share_updates = []
|
||||||
|
gathered_at = timeutils.utcnow()
|
||||||
|
for s in shares:
|
||||||
|
share_updates.append({'id': s['id'],
|
||||||
|
'used_size': 1,
|
||||||
|
'gathered_at': gathered_at})
|
||||||
|
return share_updates
|
||||||
|
|
|
@ -19,6 +19,7 @@ import os
|
||||||
import ddt
|
import ddt
|
||||||
import mock
|
import mock
|
||||||
from oslo_config import cfg
|
from oslo_config import cfg
|
||||||
|
from oslo_utils import timeutils
|
||||||
|
|
||||||
from manila.common import constants as const
|
from manila.common import constants as const
|
||||||
from manila import context
|
from manila import context
|
||||||
|
@ -591,3 +592,64 @@ class LVMShareDriverTestCase(test.TestCase):
|
||||||
update_access.assert_called_once_with(
|
update_access.assert_called_once_with(
|
||||||
self.server, self.snapshot['name'],
|
self.server, self.snapshot['name'],
|
||||||
access_rules, add_rules=add_rules, delete_rules=delete_rules))
|
access_rules, add_rules=add_rules, delete_rules=delete_rules))
|
||||||
|
|
||||||
|
@mock.patch.object(timeutils, 'utcnow', mock.Mock(
|
||||||
|
return_value='fake_date'))
|
||||||
|
def test_update_share_usage_size(self):
|
||||||
|
mount_path = self._get_mount_path(self.share)
|
||||||
|
self._os.path.exists.return_value = True
|
||||||
|
self.mock_object(
|
||||||
|
self._driver,
|
||||||
|
'_execute',
|
||||||
|
mock.Mock(return_value=(
|
||||||
|
"Mounted on Used "
|
||||||
|
+ mount_path + " 1G", None)))
|
||||||
|
|
||||||
|
update_shares = self._driver.update_share_usage_size(
|
||||||
|
self._context, [self.share, ])
|
||||||
|
self._os.path.exists.assert_called_with(mount_path)
|
||||||
|
self.assertEqual(
|
||||||
|
[{'id': 'fakeid', 'used_size': '1',
|
||||||
|
'gathered_at': 'fake_date'}],
|
||||||
|
update_shares)
|
||||||
|
self._driver._execute.assert_called_once_with(
|
||||||
|
'df', '-l', '--output=target,used',
|
||||||
|
'--block-size=g')
|
||||||
|
|
||||||
|
@mock.patch.object(timeutils, 'utcnow', mock.Mock(
|
||||||
|
return_value='fake_date'))
|
||||||
|
def test_update_share_usage_size_multiple_share(self):
|
||||||
|
share1 = fake_share(id='fakeid_get_fail', name='get_fail')
|
||||||
|
share2 = fake_share(id='fakeid_success', name='get_success')
|
||||||
|
share3 = fake_share(id='fakeid_not_exist', name='get_not_exist')
|
||||||
|
|
||||||
|
mount_path2 = self._get_mount_path(share2)
|
||||||
|
mount_path3 = self._get_mount_path(share3)
|
||||||
|
self._os.path.exists.side_effect = [True, True, False]
|
||||||
|
self.mock_object(
|
||||||
|
self._driver,
|
||||||
|
'_execute',
|
||||||
|
mock.Mock(return_value=(
|
||||||
|
"Mounted on Used "
|
||||||
|
+ mount_path2 + " 1G", None)))
|
||||||
|
|
||||||
|
update_shares = self._driver.update_share_usage_size(
|
||||||
|
self._context, [share1, share2, share3])
|
||||||
|
self._os.path.exists.assert_called_with(mount_path3)
|
||||||
|
self.assertEqual(
|
||||||
|
[{'gathered_at': 'fake_date',
|
||||||
|
'id': 'fakeid_success', 'used_size': '1'}],
|
||||||
|
update_shares)
|
||||||
|
self._driver._execute.assert_called_with(
|
||||||
|
'df', '-l', '--output=target,used',
|
||||||
|
'--block-size=g')
|
||||||
|
|
||||||
|
def test_update_share_usage_size_fail(self):
|
||||||
|
def _fake_exec(*args, **kwargs):
|
||||||
|
raise exception.ProcessExecutionError(stderr="error")
|
||||||
|
|
||||||
|
self.mock_object(self._driver, '_execute', _fake_exec)
|
||||||
|
self.assertRaises(exception.ProcessExecutionError,
|
||||||
|
self._driver.update_share_usage_size,
|
||||||
|
self._context,
|
||||||
|
[self.share])
|
||||||
|
|
|
@ -6303,6 +6303,46 @@ class ShareManagerTestCase(test.TestCase):
|
||||||
self.context, share_instance['id'],
|
self.context, share_instance['id'],
|
||||||
share_server='fake_share_server')
|
share_server='fake_share_server')
|
||||||
|
|
||||||
|
@mock.patch('manila.tests.fake_notifier.FakeNotifier._notify')
|
||||||
|
def test_update_share_usage_size(self, mock_notify):
|
||||||
|
instances = self._setup_init_mocks(setup_access_rules=False)
|
||||||
|
update_shares = [{'id': 'fake_id', 'used_size': '3',
|
||||||
|
'gathered_at': 'fake'}]
|
||||||
|
mock_notify.assert_not_called()
|
||||||
|
|
||||||
|
manager = self.share_manager
|
||||||
|
self.mock_object(manager, 'driver')
|
||||||
|
self.mock_object(manager.db, 'share_instances_get_all_by_host',
|
||||||
|
mock.Mock(return_value=instances))
|
||||||
|
self.mock_object(manager.db, 'share_instance_get',
|
||||||
|
mock.Mock(side_effect=instances))
|
||||||
|
mock_driver_call = self.mock_object(
|
||||||
|
manager.driver, 'update_share_usage_size',
|
||||||
|
mock.Mock(return_value=update_shares))
|
||||||
|
self.share_manager.update_share_usage_size(self.context)
|
||||||
|
self.assert_notify_called(mock_notify,
|
||||||
|
(['INFO', 'share.consumed.size'], ))
|
||||||
|
mock_driver_call.assert_called_once_with(
|
||||||
|
self.context, instances)
|
||||||
|
|
||||||
|
@mock.patch('manila.tests.fake_notifier.FakeNotifier._notify')
|
||||||
|
def test_update_share_usage_size_fail(self, mock_notify):
|
||||||
|
instances = self._setup_init_mocks(setup_access_rules=False)
|
||||||
|
mock_notify.assert_not_called()
|
||||||
|
|
||||||
|
self.mock_object(self.share_manager, 'driver')
|
||||||
|
self.mock_object(self.share_manager.db,
|
||||||
|
'share_instances_get_all_by_host',
|
||||||
|
mock.Mock(return_value=instances))
|
||||||
|
self.mock_object(self.share_manager.db, 'share_instance_get',
|
||||||
|
mock.Mock(side_effect=instances))
|
||||||
|
self.mock_object(
|
||||||
|
self.share_manager.driver, 'update_share_usage_size',
|
||||||
|
mock.Mock(side_effect=exception.ProcessExecutionError))
|
||||||
|
mock_log_exception = self.mock_object(manager.LOG, 'exception')
|
||||||
|
self.share_manager.update_share_usage_size(self.context)
|
||||||
|
self.assertTrue(mock_log_exception.called)
|
||||||
|
|
||||||
|
|
||||||
@ddt.ddt
|
@ddt.ddt
|
||||||
class HookWrapperTestCase(test.TestCase):
|
class HookWrapperTestCase(test.TestCase):
|
||||||
|
|
|
@ -0,0 +1,9 @@
|
||||||
|
---
|
||||||
|
features:
|
||||||
|
- Added periodic task to gather share usage size.
|
||||||
|
upgrade:
|
||||||
|
- Added enable_gathering_share_usage_size and
|
||||||
|
share_usage_size_update_interval options in the
|
||||||
|
manila.conf file to allow configuration of gathering
|
||||||
|
share usage size support and to allow configuration
|
||||||
|
of interval time of gathering share usage size.
|
Loading…
Reference in New Issue