From e3e4f4d9277c22654ce0dd9a1a0f44a67661e695 Mon Sep 17 00:00:00 2001 From: John Bresnahan Date: Wed, 21 Aug 2013 20:08:45 -1000 Subject: [PATCH] Add a storage quota This patch adds a storage quota that is applied against the sum total of a users storage consumption against all configured storage systems. A single quota is applied to all users via the configuration option 'total_storage_quota'. Most of the patch is about enforcement so when a separate service for quota management emerges in OpenStack the per user value to enforce can be obtained from that service but the enforcement code will remain the same. blueprint glance-basic-quotas docImpact Change-Id: I251832f7372c70942be6f0c6aa12285145dd7c18 --- doc/source/configuring.rst | 13 + etc/glance-api.conf | 5 + glance/api/common.py | 59 +++++ glance/api/v1/upload_utils.py | 32 +++ glance/api/v2/image_data.py | 19 ++ glance/api/v2/images.py | 6 + glance/common/config.py | 4 + glance/common/exception.py | 5 + glance/db/registry/api.py | 5 + glance/db/simple/api.py | 9 + glance/db/sqlalchemy/api.py | 19 +- glance/gateway.py | 9 +- glance/quota/__init__.py | 174 +++++++++++++ glance/tests/functional/__init__.py | 5 + glance/tests/functional/db/base.py | 73 ++++++ glance/tests/functional/db/test_registry.py | 12 + glance/tests/functional/db/test_simple.py | 8 + glance/tests/functional/v2/test_images.py | 83 +++++++ glance/tests/unit/test_quota.py | 259 ++++++++++++++++++++ glance/tests/unit/v1/test_api.py | 68 +++++ 20 files changed, 864 insertions(+), 3 deletions(-) create mode 100644 glance/quota/__init__.py create mode 100644 glance/tests/unit/test_quota.py diff --git a/doc/source/configuring.rst b/doc/source/configuring.rst index 63c2df6901..d30d34cdce 100644 --- a/doc/source/configuring.rst +++ b/doc/source/configuring.rst @@ -331,6 +331,19 @@ Maximum image size, in bytes, which can be uploaded through the Glance API serve **IMPORTANT NOTE**: this value should only be increased after careful consideration and must be set to a value under 8 EB (9223372036854775808). +Configuring Glance User Storage Quota +------------------------------------- + +The following configuration option is specified in the +``glance-api.conf`` config file in the section ``[DEFAULT]``. + +* ``user_storage_quota`` + +Optional. Default: 0 (Unlimited). + +This value specifies the maximum amount of bytes that each user can use +across all storage systems. + Configuring the Filesystem Storage Backend ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/etc/glance-api.conf b/etc/glance-api.conf index bdc878e829..d205a7e912 100644 --- a/etc/glance-api.conf +++ b/etc/glance-api.conf @@ -104,6 +104,11 @@ workers = 1 #disk_formats=ami,ari,aki,vhd,vmdk,raw,qcow2,vdi,iso +# Set a system wide quota for every user. This value is the total number +# of bytes that a user can use across all storage systems. A value of +# 0 means unlimited. +#user_storage_quota = 0 + # ================= Syslog Options ============================ # Send logs to syslog (/dev/log) instead of to file specified diff --git a/glance/api/common.py b/glance/api/common.py index 9b3b6ded25..51fc3e5efd 100644 --- a/glance/api/common.py +++ b/glance/api/common.py @@ -13,10 +13,13 @@ # License for the specific language governing permissions and limitations # under the License. +from oslo.config import cfg + from glance.common import exception from glance.openstack.common import log as logging LOG = logging.getLogger(__name__) +CONF = cfg.CONF def size_checked_iter(response, image_meta, expected_size, image_iter, @@ -76,3 +79,59 @@ def image_send_notification(bytes_written, expected_size, image_meta, request, msg = _("An error occurred during image.send" " notification: %(err)s") % locals() LOG.error(msg) + + +def get_remaining_quota(context, db_api, image_id=None): + """ + This method is called to see if the user is allowed to store an image + of the given size in glance based on their quota and current usage. + :param context: + :param db_api: The db_api in use for this configuration + :param image_id: The image that will be replaced with this new data size + :return: The number of bytes the user has remaining under their quota. + None means infinity + """ + + #NOTE(jbresnah) in the future this value will come from a call to + # keystone. + users_quota = CONF.user_storage_quota + if users_quota <= 0: + return None + + usage = db_api.user_get_storage_usage(context, + context.owner, + image_id=image_id) + return users_quota - usage + + +def check_quota(context, image_size, db_api, image_id=None): + """ + This method is called to see if the user is allowed to store an image + of the given size in glance based on their quota and current usage. + :param context: + :param image_size: The size of the image we hope to store + :param db_api: The db_api in use for this configuration + :param image_id: The image that will be replaced with this new data size + :return: + """ + + remaining = get_remaining_quota(context, db_api, image_id=image_id) + + if remaining is None: + return + + if image_size is None: + #NOTE(jbresnah) When the image size is None it means that it is + # not known. In this case the only time we will raise an + # exception is when there is no room left at all, thus we know + # it will not fit + if remaining <= 0: + raise exception.StorageQuotaFull(image_size=image_size, + remaining=remaining) + return + + if image_size > remaining: + raise exception.StorageQuotaFull(image_size=image_size, + remaining=remaining) + + return remaining diff --git a/glance/api/v1/upload_utils.py b/glance/api/v1/upload_utils.py index 360d21f364..47124e3550 100644 --- a/glance/api/v1/upload_utils.py +++ b/glance/api/v1/upload_utils.py @@ -21,6 +21,7 @@ import webob.exc from glance.common import exception from glance.openstack.common import excutils from glance.common import utils +import glance.db import glance.openstack.common.log as logging import glance.registry.client.v1.api as registry import glance.store @@ -79,7 +80,16 @@ def upload_data_to_store(req, image_meta, image_data, store, notifier): Upload image data to the store and cleans up on error. """ image_id = image_meta['id'] + + db_api = glance.db.get_api() + image_size = image_meta.get('size', None) + try: + remaining = glance.api.common.check_quota( + req.context, image_size, db_api, image_id=image_id) + if remaining is not None: + image_data = utils.LimitingReader(image_data, remaining) + (location, size, checksum, @@ -89,6 +99,18 @@ def upload_data_to_store(req, image_meta, image_data, store, notifier): image_meta['size'], store) + try: + # recheck the quota in case there were simultaneous uploads that + # did not provide the size + glance.api.common.check_quota( + req.context, size, db_api, image_id=image_id) + except exception.StorageQuotaFull: + LOG.info(_('Cleaning up %s after exceeding the quota %s') + % image_id) + glance.store.safe_delete_from_backend( + location, req.context, image_meta['id']) + raise + def _kill_mismatched(image_meta, attr, actual): supplied = image_meta.get(attr) if supplied and supplied != actual: @@ -182,6 +204,16 @@ def upload_data_to_store(req, image_meta, image_data, store, notifier): request=req, content_type='text/plain') + except exception.StorageQuotaFull as e: + msg = (_("Denying attempt to upload image because it exceeds the ." + "quota: %s") % e) + LOG.info(msg) + safe_kill(req, image_id) + notifier.error('image.upload', msg) + raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg, + request=req, + content_type='text/plain') + except webob.exc.HTTPError: #NOTE(bcwaldon): Ideally, we would just call 'raise' here, # but something in the above function calls is affecting the diff --git a/glance/api/v2/image_data.py b/glance/api/v2/image_data.py index 7d04b5ab07..adfdf91507 100644 --- a/glance/api/v2/image_data.py +++ b/glance/api/v2/image_data.py @@ -15,6 +15,7 @@ import webob.exc +import glance.api.common import glance.api.policy from glance.common import exception from glance.common import utils @@ -76,6 +77,24 @@ class ImageDataController(object): raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg, request=req) + except exception.StorageFull as e: + msg = _("Image storage media is full: %s") % e + LOG.error(msg) + raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg, + request=req) + + except exception.StorageQuotaFull as e: + msg = _("Image exceeds the storage quota: %s") % e + LOG.error(msg) + raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg, + request=req) + + except exception.ImageSizeLimitExceeded as e: + msg = _("The incoming image is too large: %") % e + LOG.error(msg) + raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg, + request=req) + except exception.StorageWriteDenied as e: msg = _("Insufficient permissions on image storage media: %s") % e LOG.error(msg) diff --git a/glance/api/v2/images.py b/glance/api/v2/images.py index 4cdb7bac53..52890a18a9 100644 --- a/glance/api/v2/images.py +++ b/glance/api/v2/images.py @@ -122,6 +122,12 @@ class ImagesController(object): raise webob.exc.HTTPNotFound(explanation=msg) except exception.Forbidden as e: raise webob.exc.HTTPForbidden(explanation=unicode(e)) + except exception.StorageQuotaFull as e: + msg = (_("Denying attempt to upload image because it exceeds the ." + "quota: %s") % e) + LOG.info(msg) + raise webob.exc.HTTPRequestEntityTooLarge( + explanation=msg, request=req, content_type='text/plain') return image diff --git a/glance/common/config.py b/glance/common/config.py index 68826cb490..d01e781441 100644 --- a/glance/common/config.py +++ b/glance/common/config.py @@ -65,6 +65,10 @@ common_opts = [ cfg.IntOpt('image_size_cap', default=1099511627776, help=_("Maximum size of image a user can upload in bytes. " "Defaults to 1099511627776 bytes (1 TB).")), + cfg.IntOpt('user_storage_quota', default=0, + help=_("Set a system wide quota for every user. This value is " + "the total number of bytes that a user can use across " + "all storage systems. A value of 0 means unlimited.")), cfg.BoolOpt('enable_v1_api', default=True, help=_("Deploy the v1 OpenStack Images API. ")), cfg.BoolOpt('enable_v2_api', default=True, diff --git a/glance/common/exception.py b/glance/common/exception.py index 62570d1315..051500f12d 100644 --- a/glance/common/exception.py +++ b/glance/common/exception.py @@ -86,6 +86,11 @@ class StorageFull(GlanceException): message = _("There is not enough disk space on the image storage media.") +class StorageQuotaFull(GlanceException): + message = _("The size of the data %(image_size)s will exceed the limit. " + "%(remaining)s bytes remaining.") + + class StorageWriteDenied(GlanceException): message = _("Permission to write image storage media denied.") diff --git a/glance/db/registry/api.py b/glance/db/registry/api.py index 8f51d21cde..485ebce710 100644 --- a/glance/db/registry/api.py +++ b/glance/db/registry/api.py @@ -216,3 +216,8 @@ def image_tag_delete(client, image_id, value, session=None): def image_tag_get_all(client, image_id, session=None): """Get a list of tags for a specific image.""" return client.image_tag_get_all(image_id=image_id) + + +@_get_client +def user_get_storage_usage(client, owner_id, image_id=None, session=None): + return client.user_get_storage_usage(owner_id=owner_id, image_id=image_id) diff --git a/glance/db/simple/api.py b/glance/db/simple/api.py index 4d24b8e1d8..ff0c5b9dc2 100644 --- a/glance/db/simple/api.py +++ b/glance/db/simple/api.py @@ -648,3 +648,12 @@ def is_image_visible(context, image, status=None): # Private image return False + + +def user_get_storage_usage(context, owner_id, image_id=None, session=None): + images = image_get_all(context, filters={'owner': owner_id}) + total = 0 + for image in images: + if image['id'] != image_id: + total = total + (image['size'] * len(image['locations'])) + return total diff --git a/glance/db/sqlalchemy/api.py b/glance/db/sqlalchemy/api.py index 4b3215b640..855e33ae0b 100644 --- a/glance/db/sqlalchemy/api.py +++ b/glance/db/sqlalchemy/api.py @@ -688,6 +688,17 @@ def _drop_protected_attrs(model_class, values): del values[attr] +def _image_get_disk_usage_by_owner(owner, session, image_id=None): + query = session.query(models.Image) + query = query.filter(models.Image.owner == owner) + if image_id is not None: + query = query.filter(models.Image.id != image_id) + query = query.filter(models.Image.size > 0) + images = query.all() + total = sum([i.size * len(i.locations) for i in images]) + return total + + def _validate_image(values): """ Validates the incoming data and raises a Invalid exception @@ -695,7 +706,6 @@ def _validate_image(values): :param values: Mapping of image metadata to check """ - status = values.get('status') status = values.get('status', None) if not status: @@ -1093,3 +1103,10 @@ def image_tag_get_all(context, image_id, session=None): .order_by(sqlalchemy.asc(models.ImageTag.created_at))\ .all() return [tag['value'] for tag in tags] + + +def user_get_storage_usage(context, owner_id, image_id=None, session=None): + session = session or _get_session() + total_size = _image_get_disk_usage_by_owner( + owner_id, session, image_id=image_id) + return total_size diff --git a/glance/gateway.py b/glance/gateway.py index 4bebb6fa37..fb3b1c184b 100644 --- a/glance/gateway.py +++ b/glance/gateway.py @@ -18,6 +18,7 @@ from glance.api import policy import glance.db import glance.domain import glance.notifier +import glance.quota import glance.store @@ -34,8 +35,10 @@ class Gateway(object): image_factory = glance.domain.ImageFactory() store_image_factory = glance.store.ImageFactoryProxy( image_factory, context, self.store_api) + quota_image_factory = glance.quota.ImageFactoryProxy( + store_image_factory, context, self.db_api) policy_image_factory = policy.ImageFactoryProxy( - store_image_factory, context, self.policy) + quota_image_factory, context, self.policy) notifier_image_factory = glance.notifier.ImageFactoryProxy( policy_image_factory, context, self.notifier) authorized_image_factory = authorization.ImageFactoryProxy( @@ -54,8 +57,10 @@ class Gateway(object): image_repo = glance.db.ImageRepo(context, self.db_api) store_image_repo = glance.store.ImageRepoProxy( image_repo, context, self.store_api) + quota_image_repo = glance.quota.ImageRepoProxy( + store_image_repo, context, self.db_api) policy_image_repo = policy.ImageRepoProxy( - store_image_repo, context, self.policy) + quota_image_repo, context, self.policy) notifier_image_repo = glance.notifier.ImageRepoProxy( policy_image_repo, context, self.notifier) authorized_image_repo = authorization.ImageRepoProxy( diff --git a/glance/quota/__init__.py b/glance/quota/__init__.py new file mode 100644 index 0000000000..6227a1adf9 --- /dev/null +++ b/glance/quota/__init__.py @@ -0,0 +1,174 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013, Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import glance.api.common +import glance.common.exception as exception +from glance.common import utils +import glance.domain +import glance.domain.proxy +import glance.openstack.common.log as logging + + +LOG = logging.getLogger(__name__) + + +class ImageRepoProxy(glance.domain.proxy.Repo): + + def __init__(self, image_repo, context, db_api): + self.image_repo = image_repo + self.db_api = db_api + proxy_kwargs = {'db_api': db_api, 'context': context} + super(ImageRepoProxy, self).__init__(image_repo, + item_proxy_class=ImageProxy, + item_proxy_kwargs=proxy_kwargs) + + +class ImageFactoryProxy(glance.domain.proxy.ImageFactory): + def __init__(self, factory, context, db_api): + proxy_kwargs = {'db_api': db_api, 'context': context} + super(ImageFactoryProxy, self).__init__(factory, + proxy_class=ImageProxy, + proxy_kwargs=proxy_kwargs) + + +class QuotaImageLocationsProxy(object): + + def __init__(self, image, context, db_api): + self.image = image + self.context = context + self.db_api = db_api + self.locations = image.locations + + def __cast__(self, *args, **kwargs): + return self.locations.__cast__(*args, **kwargs) + + def __contains__(self, *args, **kwargs): + return self.locations.__contains__(*args, **kwargs) + + def __delitem__(self, *args, **kwargs): + return self.locations.__delitem__(*args, **kwargs) + + def __delslice__(self, *args, **kwargs): + return self.locations.__delslice__(*args, **kwargs) + + def __eq__(self, other): + return self.locations == other + + def __getitem__(self, *args, **kwargs): + return self.locations.__getitem__(*args, **kwargs) + + def __iadd__(self, other): + if not hasattr(other, '__iter__'): + raise TypeError() + self._check_quota(len(list(other))) + return self.locations.__iadd__(other) + + def __iter__(self, *args, **kwargs): + return self.locations.__iter__(*args, **kwargs) + + def __len__(self, *args, **kwargs): + return self.locations.__len__(*args, **kwargs) + + def __setitem__(self, key, value): + return self.locations.__setitem__(key, value) + + def count(self, *args, **kwargs): + return self.locations.count(*args, **kwargs) + + def index(self, *args, **kwargs): + return self.locations.index(*args, **kwargs) + + def pop(self, *args, **kwargs): + return self.locations.pop(*args, **kwargs) + + def remove(self, *args, **kwargs): + return self.locations.remove(*args, **kwargs) + + def reverse(self, *args, **kwargs): + return self.locations.reverse(*args, **kwargs) + + def __getitem__(self, *args, **kwargs): + return self.locations.__getitem__(*args, **kwargs) + + def _check_quota(self, count): + glance.api.common.check_quota( + self.context, self.image.size * count, self.db_api) + + def append(self, object): + self._check_quota(1) + return self.locations.append(object) + + def insert(self, index, object): + self._check_quota(1) + return self.locations.insert(index, object) + + def extend(self, iter): + self._check_quota(len(list(iter))) + return self.locations.extend(iter) + + +class ImageProxy(glance.domain.proxy.Image): + + def __init__(self, image, context, db_api): + self.image = image + self.context = context + self.db_api = db_api + super(ImageProxy, self).__init__(image) + + def set_data(self, data, size=None): + remaining = glance.api.common.check_quota( + self.context, size, self.db_api, image_id=self.image.image_id) + if remaining is not None: + # NOTE(jbresnah) we are trying to enforce a quota, put a limit + # reader on the data + data = utils.LimitingReader(data, remaining) + try: + self.image.set_data(data, size=size) + except exception.ImageSizeLimitExceeded as ex: + raise exception.StorageQuotaFull(image_size=size, + remaining=remaining) + + # NOTE(jbresnah) If two uploads happen at the same time and neither + # properly sets the size attribute than there is a race condition + # that will allow for the quota to be broken. Thus we must recheck + # the quota after the upload and thus after we know the size + try: + glance.api.common.check_quota( + self.context, self.image.size, self.db_api, + image_id=self.image.image_id) + except exception.StorageQuotaFull: + LOG.info(_('Cleaning up %s after exceeding the quota.') + % self.image.image_id) + location = self.image.locations[0]['url'] + glance.store.safe_delete_from_backend( + location, self.context, self.image.image_id) + raise + + @property + def locations(self): + return QuotaImageLocationsProxy(self.image, + self.context, + self.db_api) + + @locations.setter + def locations(self, value): + if not isinstance(value, (list, QuotaImageLocationsProxy)): + raise exception.Invalid(_('Invalid locations: %s') % value) + glance.api.common.check_quota( + self.context, self.image.size * len(value), self.db_api, + image_id=self.image.image_id) + self.image.locations = value diff --git a/glance/tests/functional/__init__.py b/glance/tests/functional/__init__.py index 71a13be3e1..4acc7c242d 100644 --- a/glance/tests/functional/__init__.py +++ b/glance/tests/functional/__init__.py @@ -314,6 +314,7 @@ class ApiServer(Server): default_sql_connection = 'sqlite:////%s/tests.sqlite' % self.test_dir self.sql_connection = os.environ.get('GLANCE_TEST_SQL_CONNECTION', default_sql_connection) + self.user_storage_quota = 0 self.conf_base = """[DEFAULT] verbose = %(verbose)s @@ -360,6 +361,7 @@ db_auto_create = False sql_connection = %(sql_connection)s show_image_direct_url = %(show_image_direct_url)s show_multiple_locations = %(show_multiple_locations)s +user_storage_quota = %(user_storage_quota)s enable_v1_api = %(enable_v1_api)s enable_v2_api= %(enable_v2_api)s [paste_deploy] @@ -448,6 +450,8 @@ class RegistryServer(Server): self.owner_is_tenant = True self.workers = 0 self.api_version = 1 + self.user_storage_quota = 0 + self.conf_base = """[DEFAULT] verbose = %(verbose)s debug = %(debug)s @@ -461,6 +465,7 @@ api_limit_max = 1000 limit_param_default = 25 owner_is_tenant = %(owner_is_tenant)s workers = %(workers)s +user_storage_quota = %(user_storage_quota)s [paste_deploy] flavor = %(deployment_flavor)s """ diff --git a/glance/tests/functional/db/base.py b/glance/tests/functional/db/base.py index 5c391dd23d..11dfab4833 100644 --- a/glance/tests/functional/db/base.py +++ b/glance/tests/functional/db/base.py @@ -1092,6 +1092,79 @@ class DriverTests(object): self.assertEqual(0, len(self.db_api.image_member_find(self.context))) +class DriverQuotaTests(test_utils.BaseTestCase): + + def setUp(self): + super(DriverQuotaTests, self).setUp() + self.owner_id1 = uuidutils.generate_uuid() + self.context1 = context.RequestContext( + is_admin=False, auth_tok='user:user:user', user=self.owner_id1) + self.db_api = db_tests.get_db(self.config) + db_tests.reset_db(self.db_api) + self.addCleanup(timeutils.clear_time_override) + dt1 = timeutils.utcnow() + dt2 = dt1 + datetime.timedelta(microseconds=5) + fixtures = [ + { + 'id': UUID1, + 'created_at': dt1, + 'updated_at': dt1, + 'size': 13, + 'owner': self.owner_id1, + }, + { + 'id': UUID2, + 'created_at': dt1, + 'updated_at': dt2, + 'size': 17, + 'owner': self.owner_id1, + }, + { + 'id': UUID3, + 'created_at': dt2, + 'updated_at': dt2, + 'size': 7, + 'owner': self.owner_id1, + }, + ] + self.owner1_fixtures = [ + build_image_fixture(**fixture) for fixture in fixtures] + + for fixture in self.owner1_fixtures: + self.db_api.image_create(self.context1, fixture) + + def test_storage_quota(self): + total = reduce(lambda x, y: x + y, + [f['size'] for f in self.owner1_fixtures]) + x = self.db_api.user_get_storage_usage(self.context1, self.owner_id1) + self.assertEqual(total, x) + + def test_storage_quota_without_image_id(self): + total = reduce(lambda x, y: x + y, + [f['size'] for f in self.owner1_fixtures]) + total = total - self.owner1_fixtures[0]['size'] + x = self.db_api.user_get_storage_usage( + self.context1, self.owner_id1, + image_id=self.owner1_fixtures[0]['id']) + self.assertEqual(total, x) + + def test_storage_quota_multiple_locations(self): + dt1 = timeutils.utcnow() + sz = 53 + new_fixture_dict = {'id': 'SOMEID', 'created_at': dt1, + 'updated_at': dt1, 'size': sz, + 'owner': self.owner_id1} + new_fixture = build_image_fixture(**new_fixture_dict) + new_fixture['locations'].append({'url': 'file:///some/path/file', + 'metadata': {}}) + self.db_api.image_create(self.context1, new_fixture) + + total = reduce(lambda x, y: x + y, + [f['size'] for f in self.owner1_fixtures]) + (sz * 2) + x = self.db_api.user_get_storage_usage(self.context1, self.owner_id1) + self.assertEqual(total, x) + + class TestVisibility(test_utils.BaseTestCase): def setUp(self): super(TestVisibility, self).setUp() diff --git a/glance/tests/functional/db/test_registry.py b/glance/tests/functional/db/test_registry.py index c66ab77c85..07abfa9252 100644 --- a/glance/tests/functional/db/test_registry.py +++ b/glance/tests/functional/db/test_registry.py @@ -72,3 +72,15 @@ class TestRegistryDriver(base.TestDriver, def tearDown(self): self.registry_server.stop() super(TestRegistryDriver, self).tearDown() + + +class TestRegistryQuota(base.DriverQuotaTests, FunctionalInitWrapper): + + def setUp(self): + db_tests.load(get_db, reset_db) + super(TestRegistryQuota, self).setUp() + self.addCleanup(db_tests.reset) + + def tearDown(self): + self.registry_server.stop() + super(TestRegistryQuota, self).tearDown() diff --git a/glance/tests/functional/db/test_simple.py b/glance/tests/functional/db/test_simple.py index 293941503c..3eb209e108 100644 --- a/glance/tests/functional/db/test_simple.py +++ b/glance/tests/functional/db/test_simple.py @@ -35,6 +35,14 @@ class TestSimpleDriver(base.TestDriver, base.DriverTests): self.addCleanup(db_tests.reset) +class TestSimpleQuota(base.DriverQuotaTests): + + def setUp(self): + db_tests.load(get_db, reset_db) + super(TestSimpleQuota, self).setUp() + self.addCleanup(db_tests.reset) + + class TestSimpleVisibility(base.TestVisibility, base.VisibilityTests): def setUp(self): diff --git a/glance/tests/functional/v2/test_images.py b/glance/tests/functional/v2/test_images.py index ead659266d..4985af5c3e 100644 --- a/glance/tests/functional/v2/test_images.py +++ b/glance/tests/functional/v2/test_images.py @@ -1111,3 +1111,86 @@ class TestImageMembers(functional.FunctionalTest): self.assertEqual(404, response.status_code) self.stop_servers() + + +class TestQuotas(functional.FunctionalTest): + + def setUp(self): + super(TestQuotas, self).setUp() + self.cleanup() + self.api_server.deployment_flavor = 'noauth' + self.user_storage_quota = 100 + self.start_servers(**self.__dict__.copy()) + + def _url(self, path): + return 'http://127.0.0.1:%d%s' % (self.api_port, path) + + def _headers(self, custom_headers=None): + base_headers = { + 'X-Identity-Status': 'Confirmed', + 'X-Auth-Token': '932c5c84-02ac-4fe5-a9ba-620af0e2bb96', + 'X-User-Id': 'f9a41d13-0c13-47e9-bee2-ce4e8bfe958e', + 'X-Tenant-Id': TENANT1, + 'X-Roles': 'member', + } + base_headers.update(custom_headers or {}) + return base_headers + + def test_image_upload_under_quota(self): + # Image list should be empty + path = self._url('/v2/images') + response = requests.get(path, headers=self._headers()) + self.assertEqual(200, response.status_code) + images = json.loads(response.text)['images'] + self.assertEqual(0, len(images)) + + # Create an image (with a deployer-defined property) + path = self._url('/v2/images') + headers = self._headers({'content-type': 'application/json'}) + data = json.dumps({'name': 'image-2', + 'disk_format': 'aki', 'container_format': 'aki'}) + response = requests.post(path, headers=headers, data=data) + self.assertEqual(201, response.status_code) + image = json.loads(response.text) + image_id = image['id'] + + # upload data + data = 'x' * (self.user_storage_quota - 1) + path = self._url('/v2/images/%s/file' % image_id) + headers = self._headers({'Content-Type': 'application/octet-stream'}) + response = requests.put(path, headers=headers, data=data) + self.assertEqual(204, response.status_code) + + # Deletion should work + path = self._url('/v2/images/%s' % image_id) + response = requests.delete(path, headers=self._headers()) + self.assertEqual(204, response.status_code) + + def test_image_upload_exceed_quota(self): + # Image list should be empty + path = self._url('/v2/images') + response = requests.get(path, headers=self._headers()) + self.assertEqual(200, response.status_code) + images = json.loads(response.text)['images'] + self.assertEqual(0, len(images)) + + # Create an image (with a deployer-defined property) + path = self._url('/v2/images') + headers = self._headers({'content-type': 'application/json'}) + data = json.dumps({'name': 'image-1', 'type': 'kernel', 'foo': 'bar', + 'disk_format': 'aki', 'container_format': 'aki'}) + response = requests.post(path, headers=headers, data=data) + self.assertEqual(201, response.status_code) + image = json.loads(response.text) + image_id = image['id'] + + # upload data + data = 'x' * (self.user_storage_quota + 1) + path = self._url('/v2/images/%s/file' % image_id) + headers = self._headers({'Content-Type': 'application/octet-stream'}) + response = requests.put(path, headers=headers, data=data) + self.assertEqual(413, response.status_code) + + path = self._url('/v2/images/%s' % image_id) + response = requests.delete(path, headers=self._headers()) + self.assertEqual(204, response.status_code) diff --git a/glance/tests/unit/test_quota.py b/glance/tests/unit/test_quota.py new file mode 100644 index 0000000000..92f274b0b0 --- /dev/null +++ b/glance/tests/unit/test_quota.py @@ -0,0 +1,259 @@ +# Copyright 2013, Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mox + +from glance.common import exception +import glance.quota +import glance.store +from glance.tests.unit import utils as unit_test_utils +from glance.tests import utils as test_utils + +UUID1 = 'c80a1a6c-bd1f-41c5-90ee-81afedb1d58d' + + +class ImageRepoStub(object): + def get(self, *args, **kwargs): + return 'image_from_get' + + def save(self, *args, **kwargs): + return 'image_from_save' + + def add(self, *args, **kwargs): + return 'image_from_add' + + def list(self, *args, **kwargs): + return ['image_from_list_0', 'image_from_list_1'] + + +class ImageStub(object): + def __init__(self, image_id, visibility='private'): + self.image_id = image_id + self.visibility = visibility + self.status = 'active' + + def delete(self): + self.status = 'deleted' + + +class ImageFactoryStub(object): + def new_image(self, image_id=None, name=None, visibility='private', + min_disk=0, min_ram=0, protected=False, owner=None, + disk_format=None, container_format=None, + extra_properties=None, tags=None, **other_args): + self.visibility = visibility + return 'new_image' + + +class FakeContext(object): + owner = 'someone' + is_admin = False + + +class FakeImage(object): + size = None + image_id = 'someid' + locations = [{'url': 'file:///not/a/path', 'metadata': {}}] + + def set_data(self, data, size=None): + self.size = 0 + for d in data: + self.size = self. size + len(d) + + +class TestImageQuota(test_utils.BaseTestCase): + def setUp(self): + super(TestImageQuota, self).setUp() + self.mox = mox.Mox() + + def tearDown(self): + super(TestImageQuota, self).tearDown() + self.mox.UnsetStubs() + + def _get_image(self, location_count=1, image_size=10): + context = FakeContext() + db_api = unit_test_utils.FakeDB() + base_image = FakeImage() + base_image.image_id = 'xyz' + base_image.size = image_size + image = glance.quota.ImageProxy(base_image, context, db_api) + locations = [] + for i in range(location_count): + locations.append({'url': 'file:///g/there/it/is%d' % i, + 'metadata': {}}) + image_values = {'id': 'xyz', 'owner': context.owner, + 'status': 'active', 'size': image_size, + 'locations': locations} + db_api.image_create(context, image_values) + return image + + def test_quota_allowed(self): + quota = 10 + self.config(user_storage_quota=quota) + context = FakeContext() + db_api = unit_test_utils.FakeDB() + base_image = FakeImage() + base_image.image_id = 'id' + image = glance.quota.ImageProxy(base_image, context, db_api) + data = '*' * quota + base_image.set_data(data, size=None) + image.set_data(data) + self.assertEqual(quota, base_image.size) + + def _quota_exceeded_size(self, quota, data, + deleted=True, size=None): + self.config(user_storage_quota=quota) + context = FakeContext() + db_api = unit_test_utils.FakeDB() + base_image = FakeImage() + base_image.image_id = 'id' + image = glance.quota.ImageProxy(base_image, context, db_api) + + if deleted: + self.mox.StubOutWithMock(glance.store, 'safe_delete_from_backend') + glance.store.safe_delete_from_backend( + base_image.locations[0]['url'], + context, + image.image_id) + + self.mox.ReplayAll() + self.assertRaises(exception.StorageQuotaFull, + image.set_data, + data, + size=size) + self.mox.VerifyAll() + + def test_quota_exceeded_no_size(self): + quota = 10 + data = '*' * (quota + 1) + self._quota_exceeded_size(quota, data) + + def test_quota_exceeded_with_right_size(self): + quota = 10 + data = '*' * (quota + 1) + self._quota_exceeded_size(quota, data, size=len(data), deleted=False) + + def test_quota_exceeded_with_lie_size(self): + quota = 10 + data = '*' * (quota + 1) + self._quota_exceeded_size(quota, data, deleted=False, size=quota - 1) + + def test_append_location(self): + new_location = {'url': 'file:///a/path', 'metadata': {}} + image = self._get_image() + pre_add_locations = image.locations[:] + image.locations.append(new_location) + pre_add_locations.append(new_location) + self.assertEqual(image.locations, pre_add_locations) + + def test_insert_location(self): + new_location = {'url': 'file:///a/path', 'metadata': {}} + image = self._get_image() + pre_add_locations = image.locations[:] + image.locations.insert(0, new_location) + pre_add_locations.insert(0, new_location) + self.assertEqual(image.locations, pre_add_locations) + + def test_extend_location(self): + new_location = {'url': 'file:///a/path', 'metadata': {}} + image = self._get_image() + pre_add_locations = image.locations[:] + image.locations.extend([new_location]) + pre_add_locations.extend([new_location]) + self.assertEqual(image.locations, pre_add_locations) + + def test_iadd_location(self): + new_location = {'url': 'file:///a/path', 'metadata': {}} + image = self._get_image() + pre_add_locations = image.locations[:] + image.locations += [new_location] + pre_add_locations += [new_location] + self.assertEqual(image.locations, pre_add_locations) + + def test_set_location(self): + new_location = {'url': 'file:///a/path', 'metadata': {}} + image = self._get_image() + image.locations = [new_location] + self.assertEqual(image.locations, [new_location]) + + def test_exceed_append_location(self): + image_size = 10 + max_images = 2 + quota = image_size * max_images + self.config(user_storage_quota=quota) + image = self._get_image(image_size=image_size, + location_count=max_images) + self.assertRaises(exception.StorageQuotaFull, + image.locations.append, + {'url': 'file:///a/path', 'metadata': {}}) + + def test_exceed_append_location(self): + image_size = 10 + max_images = 2 + quota = image_size * max_images + self.config(user_storage_quota=quota) + image = self._get_image(image_size=image_size, + location_count=max_images) + self.assertRaises(exception.StorageQuotaFull, + image.locations.insert, + 0, + {'url': 'file:///a/path', 'metadata': {}}) + + def test_exceed_extend_location(self): + image_size = 10 + max_images = 2 + quota = image_size * max_images + self.config(user_storage_quota=quota) + image = self._get_image(image_size=image_size, + location_count=max_images) + self.assertRaises(exception.StorageQuotaFull, + image.locations.extend, + [{'url': 'file:///a/path', 'metadata': {}}]) + + def test_set_location_under(self): + image_size = 10 + max_images = 1 + quota = image_size * max_images + self.config(user_storage_quota=quota) + image = self._get_image(image_size=image_size, + location_count=max_images) + image.locations = [{'url': 'file:///a/path', 'metadata': {}}] + + def test_set_location_exceed(self): + image_size = 10 + max_images = 1 + quota = image_size * max_images + self.config(user_storage_quota=quota) + image = self._get_image(image_size=image_size, + location_count=max_images) + try: + image.locations = [{'url': 'file:///a/path', 'metadata': {}}, + {'url': 'file:///a/path2', 'metadata': {}}] + self.fail('Should have raised the quota exception') + except exception.StorageQuotaFull: + pass + + def test_iadd_location_exceed(self): + image_size = 10 + max_images = 1 + quota = image_size * max_images + self.config(user_storage_quota=quota) + image = self._get_image(image_size=image_size, + location_count=max_images) + try: + image.locations += [{'url': 'file:///a/path', 'metadata': {}}] + self.fail('Should have raised the quota exception') + except exception.StorageQuotaFull: + pass diff --git a/glance/tests/unit/v1/test_api.py b/glance/tests/unit/v1/test_api.py index df0e34014d..773a3e4d69 100644 --- a/glance/tests/unit/v1/test_api.py +++ b/glance/tests/unit/v1/test_api.py @@ -416,6 +416,74 @@ class TestGlanceAPI(base.IsolatedUnitTest): res = req.get_response(self.api) self.assertEquals(res.status_int, 400) + def test_add_image_size_header_exceed_quota(self): + quota = 500 + self.config(user_storage_quota=quota) + fixture_headers = {'x-image-meta-size': quota + 1, + 'x-image-meta-name': 'fake image #3', + 'x-image-meta-container_format': 'bare', + 'x-image-meta-disk_format': 'qcow2', + 'content-type': 'application/octet-stream', + } + + req = webob.Request.blank("/images") + req.method = 'POST' + for k, v in fixture_headers.iteritems(): + req.headers[k] = v + req.body = 'X' * (quota + 1) + res = req.get_response(self.api) + self.assertEquals(res.status_int, 413) + + def test_add_image_size_data_exceed_quota(self): + quota = 500 + self.config(user_storage_quota=quota) + fixture_headers = { + 'x-image-meta-name': 'fake image #3', + 'x-image-meta-container_format': 'bare', + 'x-image-meta-disk_format': 'qcow2', + 'content-type': 'application/octet-stream', + } + + req = webob.Request.blank("/images") + req.method = 'POST' + + req.body = 'X' * (quota + 1) + for k, v in fixture_headers.iteritems(): + req.headers[k] = v + + res = req.get_response(self.api) + self.assertEquals(res.status_int, 413) + + def test_add_image_size_data_exceed_quota_readd(self): + quota = 500 + self.config(user_storage_quota=quota) + fixture_headers = { + 'x-image-meta-name': 'fake image #3', + 'x-image-meta-container_format': 'bare', + 'x-image-meta-disk_format': 'qcow2', + 'content-type': 'application/octet-stream', + } + + req = webob.Request.blank("/images") + req.method = 'POST' + req.body = 'X' * (quota + 1) + for k, v in fixture_headers.iteritems(): + req.headers[k] = v + + res = req.get_response(self.api) + self.assertEquals(res.status_int, 413) + + used_size = sum([f['size'] for f in self.FIXTURES]) + + req = webob.Request.blank("/images") + req.method = 'POST' + req.body = 'X' * (quota - used_size) + for k, v in fixture_headers.iteritems(): + req.headers[k] = v + + res = req.get_response(self.api) + self.assertEquals(res.status_int, 201) + def _add_check_no_url_info(self): fixture_headers = {'x-image-meta-disk-format': 'ami',