From 1175b0f7c1bc4f042d611396ecee5720e1b2d7bd Mon Sep 17 00:00:00 2001 From: Dan Smith Date: Fri, 4 Jun 2021 08:00:41 -0700 Subject: [PATCH] Add Enforcer.calculate_usage() In multiple situations, it is necessary to be able to probe the limits set for a project without actually enforcing. Examples: 1. Exposing a usage API where we want to not only report the current usage, but the limit as well. Otherwise clients have to do their own calls to keystone and correlation to get a single integer limit value, which we should be able to expose for them. 2. When checking quota as part of a long-running process of consuming an unbounded data stream, we need to be able to determine how much quota remains so that we can stop the transfer if we exceed the limit. Without this, we have to periodically call to keystone during the transfer, which is expensive and could fail. This patch adds a calculate_usage() method to the Enforcer which calculates the usage using the enforcement model and returns a mapping of resource names to namedtuples that contain limit and usage information. Change-Id: Ic0632cc5ec52aefb85a04f879651963bfa54dcbe --- doc/source/user/usage.rst | 34 ++++++++++++++++++ oslo_limit/limit.py | 64 ++++++++++++++++++++++++++++++++-- oslo_limit/tests/test_limit.py | 43 +++++++++++++++++++++++ 3 files changed, 138 insertions(+), 3 deletions(-) diff --git a/doc/source/user/usage.rst b/doc/source/user/usage.rst index 5ca00ff..8eca3e5 100644 --- a/doc/source/user/usage.rst +++ b/doc/source/user/usage.rst @@ -138,3 +138,37 @@ Here is a simple usage of limit enforcement # What to do in case of limit exception, e contain a list of # resource over quota logging.error(e) + +Check a limit +------------- + +Another usage pattern is to check a limit and usage for a given +project, outside the scope of enforcement. This may be useful in a +reporting API to be able to expose to a user the limit and usage +information that the enforcer would use to judge a resource +consumption event. + +.. note:: + This should ideally not be used to provide your own enforcement of + limits, but rather for reporting or planning purposes. + +Here is a simple usage of limit reporting + +.. code-block:: python + + import logging + + from oslo_limit import limit + + # Callback function who need to return resource usage for each + # resource asked in resources_names, for a given project_id + def callback(project_id, resource_names): + return {x: get_resource_usage_by_project(x, project_id) for x in resource_names} + + enforcer = limit.Enforcer(callback) + usage = enforcer.calculate_usage('project_uuid', ['my_resource']) + logging.info('%s using %i out of %i allowed %s resource' % ( + 'project_uuid', + usage['my_resource'].usage, + usage['my_resource'].limit, + 'my_resource')) diff --git a/oslo_limit/limit.py b/oslo_limit/limit.py index ad7de98..d63f430 100644 --- a/oslo_limit/limit.py +++ b/oslo_limit/limit.py @@ -12,6 +12,8 @@ # License for the specific language governing permissions and limitations # under the License. +from collections import namedtuple + from keystoneauth1 import exceptions as ksa_exceptions from keystoneauth1 import loading from openstack import connection @@ -27,6 +29,9 @@ _SDK_CONNECTION = None opts.register_opts(CONF) +ProjectUsage = namedtuple('ProjectUsage', ['limit', 'usage']) + + def _get_keystone_connection(): global _SDK_CONNECTION if not _SDK_CONNECTION: @@ -124,6 +129,46 @@ class Enforcer(object): self.model.enforce(project_id, deltas) + def calculate_usage(self, project_id, resources_to_check): + """Calculate resource usage and limits for resources_to_check. + + From the list of resources_to_check, we collect the project's + limit and current usage for each, exactly like we would for + enforce(). This is useful for reporting current project usage + and limits in a situation where enforcement is not desired. + + This should *not* be used to conduct custom enforcement, but + rather only for reporting. + + :param project_id: The project for which to check usage and limits. + :type project_id: string + :param resources_to_check: A list of resource names to query. + :type resources_to_check: list + :returns: A dictionary of name:limit.ProjectUsage for the + requested names against the provided project. + """ + if not project_id or not isinstance(project_id, str): + msg = 'project_id must be a non-empty string.' + raise ValueError(msg) + + msg = ('resources_to_check must be non-empty sequence of ' + 'resource name strings') + try: + if len(resources_to_check) == 0: + raise ValueError(msg) + except TypeError: + raise ValueError(msg) + + for resource_name in resources_to_check: + if not isinstance(resource_name, str): + raise ValueError(msg) + + limits = self.model.get_project_limits(project_id, resources_to_check) + usage = self.model.get_project_usage(project_id, resources_to_check) + + return {resource: ProjectUsage(limit, usage[resource]) + for resource, limit in dict(limits).items()} + class _FlatEnforcer(object): @@ -133,14 +178,21 @@ class _FlatEnforcer(object): self._usage_callback = usage_callback self._utils = _EnforcerUtils() + def get_project_limits(self, project_id, resources_to_check): + return self._utils.get_project_limits(project_id, resources_to_check) + + def get_project_usage(self, project_id, resources_to_check): + return self._usage_callback(project_id, resources_to_check) + def enforce(self, project_id, deltas): resources_to_check = list(deltas.keys()) # Always check the limits in the same order, for predictable errors resources_to_check.sort() - project_limits = self._utils.get_project_limits(project_id, - resources_to_check) - current_usage = self._usage_callback(project_id, resources_to_check) + project_limits = self.get_project_limits(project_id, + resources_to_check) + current_usage = self.get_project_usage(project_id, + resources_to_check) self._utils.enforce_limits(project_id, project_limits, current_usage, deltas) @@ -153,6 +205,12 @@ class _StrictTwoLevelEnforcer(object): def __init__(self, usage_callback): self._usage_callback = usage_callback + def get_project_limits(self, project_id, resources_to_check): + raise NotImplementedError() + + def get_project_usage(self, project_id, resources_to_check): + raise NotImplementedError() + def enforce(self, project_id, deltas): raise NotImplementedError() diff --git a/oslo_limit/tests/test_limit.py b/oslo_limit/tests/test_limit.py index a84a35c..c229b6d 100644 --- a/oslo_limit/tests/test_limit.py +++ b/oslo_limit/tests/test_limit.py @@ -125,6 +125,49 @@ class TestEnforcer(base.BaseTestCase): mock_enforce.assert_called_once_with(project_id, deltas) + @mock.patch.object(limit._EnforcerUtils, "get_project_limits") + def test_calculate_usage(self, mock_get_limits): + mock_usage = mock.MagicMock() + mock_usage.return_value = {'a': 1, 'b': 2} + + project_id = uuid.uuid4().hex + mock_get_limits.return_value = [('a', 10), ('b', 5)] + + expected = { + 'a': limit.ProjectUsage(10, 1), + 'b': limit.ProjectUsage(5, 2), + } + + enforcer = limit.Enforcer(mock_usage) + self.assertEqual(expected, enforcer.calculate_usage(project_id, + ['a', 'b'])) + + def test_calculate_usage_bad_params(self): + enforcer = limit.Enforcer(mock.MagicMock()) + + # Non-string project_id + self.assertRaises(ValueError, + enforcer.calculate_usage, + None, ['foo']) + self.assertRaises(ValueError, + enforcer.calculate_usage, + 123, ['foo']) + + # Zero-length resources_to_check + self.assertRaises(ValueError, + enforcer.calculate_usage, + 'project', []) + + # Non-sequence resources_to_check + self.assertRaises(ValueError, + enforcer.calculate_usage, + 'project', 123) + + # Invalid non-string value in resources_to_check + self.assertRaises(ValueError, + enforcer.calculate_usage, + 'project', ['a', 123, 'b']) + class TestFlatEnforcer(base.BaseTestCase): def setUp(self):