From 87eae327bf66a8e6fb7e7b72da9a4e62eee65702 Mon Sep 17 00:00:00 2001 From: Erno Kuvaja Date: Tue, 18 May 2021 18:29:42 +0100 Subject: [PATCH] Cache management API endpoints This change adds the new cache API endpoints and their related new policies. Implements-bp: https://blueprints.launchpad.net/glance/+spec/cache-api Change-Id: I69162e19bf095ef11fbac56a1ea2159d1caefba7 --- api-ref/source/v2/cache-manage.inc | 84 ++++ api-ref/source/v2/images-parameters.yaml | 8 + api-ref/source/v2/index.rst | 1 + doc/source/configuration/configuring.rst | 9 +- glance/api/middleware/version_negotiation.py | 2 + glance/api/v2/cached_images.py | 132 ++++++- glance/api/v2/policy.py | 13 +- glance/api/v2/router.py | 29 ++ glance/api/versions.py | 10 +- glance/policies/__init__.py | 2 + glance/policies/base.py | 1 + glance/policies/cache.py | 75 ++++ glance/tests/functional/__init__.py | 2 + glance/tests/functional/test_api.py | 15 +- glance/tests/functional/v2/test_cache_api.py | 360 ++++++++++++++++++ glance/tests/unit/test_cached_images.py | 355 ++++++++++++++--- glance/tests/unit/test_versions.py | 80 +++- .../unit/v2/test_cache_management_api.py | 123 ++++++ glance/tests/unit/v2/test_v2_policy.py | 34 +- .../notes/cache-api-b806ccfb8c5d9bb6.yaml | 9 + 20 files changed, 1268 insertions(+), 76 deletions(-) create mode 100644 api-ref/source/v2/cache-manage.inc create mode 100644 glance/policies/cache.py create mode 100644 glance/tests/functional/v2/test_cache_api.py create mode 100644 glance/tests/unit/v2/test_cache_management_api.py create mode 100644 releasenotes/notes/cache-api-b806ccfb8c5d9bb6.yaml diff --git a/api-ref/source/v2/cache-manage.inc b/api-ref/source/v2/cache-manage.inc new file mode 100644 index 0000000000..1993cb191d --- /dev/null +++ b/api-ref/source/v2/cache-manage.inc @@ -0,0 +1,84 @@ +.. -*- rst -*- + +Cache Manage +************ + +List and manage the cache. + + +Query cache status +~~~~~~~~~~~~~~~~~~ + +.. rest_method:: GET /v2/cache/ + +Lists all images in cache or queue. +*(Since Image API v2.14)* + +Normal response codes: 200 + +Error response codes: 400, 401, 403 + + +Request +------- + +No request parameters. + + +Queue image +~~~~~~~~~~~ + +.. rest_method:: PUT /v2/cache/{image_id}/ + +Queues image for caching. +*(Since Image API v2.14)* + +Normal response codes: 200 + +Error response codes: 400, 401, 403, 404 + + +Request +------- + + - image_id: image_id-in-path + + +Delete image from cache +~~~~~~~~~~~~~~~~~~~~~~~ + +.. rest_method:: DELETE /v2/cache/{image_id}/ + +Deletes a image from cache. +*(Since Image API v2.14)* + +Normal response codes: 204 + +Error response codes: 400, 401, 403, 404 + + +Request +------- + + - image_id: image_id-in-path + + +Clear images from cache +~~~~~~~~~~~~~~~~~~~~~~~ + +.. rest_method:: DELETE /v2/cache/ + +Clears the cache and its queue. +*(Since Image API v2.14)* + +Normal response codes: 204 + +Error response codes: 400, 401, 403 + + +Request +------- + +.. rest_parameters:: images-parameters.yaml + + - x-image-cache-clear-target: cache-clear-header diff --git a/api-ref/source/v2/images-parameters.yaml b/api-ref/source/v2/images-parameters.yaml index 459a20b1c8..a99bbd6014 100644 --- a/api-ref/source/v2/images-parameters.yaml +++ b/api-ref/source/v2/images-parameters.yaml @@ -1,4 +1,12 @@ # variables in header +cache-clear-header: + description: | + A keyword indicating 'cache', 'queue' or empty string to indicate the delete + API to delete images from cache or queue or delete from both. If this header + is missing then all cached and queued images for caching will be deleted. + in: header + required: false + type: string Content-Length: description: | The length of the body in octets (8-bit bytes) diff --git a/api-ref/source/v2/index.rst b/api-ref/source/v2/index.rst index f18dfbf31a..61b2eb7cdd 100644 --- a/api-ref/source/v2/index.rst +++ b/api-ref/source/v2/index.rst @@ -33,3 +33,4 @@ Image Service API v2 (CURRENT) .. include:: discovery.inc .. include:: tasks.inc .. include:: tasks-schemas.inc +.. include:: cache-manage.inc diff --git a/doc/source/configuration/configuring.rst b/doc/source/configuration/configuring.rst index 9cb26f74a0..8d02775d2d 100644 --- a/doc/source/configuration/configuring.rst +++ b/doc/source/configuration/configuring.rst @@ -1390,8 +1390,8 @@ configuration file, select the appropriate deployment flavor like so:: [paste_deploy] flavor = caching -Enabling the Image Cache Management Middleware -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Enabling the Image Cache Management Middleware (DEPRECATED) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ There is an optional ``cachemanage`` middleware that allows you to directly interact with cache images. Use this flavor in place of the @@ -1402,6 +1402,11 @@ can chose: ``cachemanagement``, ``keystone+cachemanagement`` and [paste_deploy] flavor = keystone+cachemanagement +The new cache management endpoints were introduced in Images API v. 2.13. +If cache middleware is configured the new endpoints will be active and +there is no need to use the cachemanagement middleware unless the old +`glance-cache-manage` tooling is desired to be still used. + Configuration Options Affecting the Image Cache ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/glance/api/middleware/version_negotiation.py b/glance/api/middleware/version_negotiation.py index ffa30347c7..b989093ec3 100644 --- a/glance/api/middleware/version_negotiation.py +++ b/glance/api/middleware/version_negotiation.py @@ -83,6 +83,8 @@ class VersionNegotiationFilter(wsgi.Middleware): allowed_versions['v2.7'] = 2 allowed_versions['v2.9'] = 2 allowed_versions['v2.13'] = 2 + if CONF.image_cache_dir: + allowed_versions['v2.14'] = 2 if CONF.enabled_backends: allowed_versions['v2.8'] = 2 allowed_versions['v2.10'] = 2 diff --git a/glance/api/v2/cached_images.py b/glance/api/v2/cached_images.py index 93663df205..002b8cf9bf 100644 --- a/glance/api/v2/cached_images.py +++ b/glance/api/v2/cached_images.py @@ -17,6 +17,8 @@ Controller for Image Cache Management API """ +import glance_store +from oslo_config import cfg from oslo_log import log as logging import webob.exc @@ -24,8 +26,14 @@ from glance.api import policy from glance.api.v2 import policy as api_policy from glance.common import exception from glance.common import wsgi +import glance.db +import glance.gateway +from glance.i18n import _ from glance import image_cache +import glance.notifier + +CONF = cfg.CONF LOG = logging.getLogger(__name__) @@ -34,19 +42,36 @@ class CacheController(object): A controller for managing cached images. """ - def __init__(self): - self.cache = image_cache.ImageCache() - self.policy = policy.Enforcer() + def __init__(self, db_api=None, policy_enforcer=None, notifier=None, + store_api=None): + if not CONF.image_cache_dir: + self.cache = None + else: + self.cache = image_cache.ImageCache() - def _enforce(self, req): - """Authorize request against 'manage_image_cache' policy""" + self.policy = policy_enforcer or policy.Enforcer() + self.db_api = db_api or glance.db.get_api() + self.notifier = notifier or glance.notifier.Notifier() + self.store_api = store_api or glance_store + self.gateway = glance.gateway.Gateway(self.db_api, self.store_api, + self.notifier, self.policy) + + def _enforce(self, req, image=None, new_policy=None): + """Authorize request against given policy""" + if not new_policy: + new_policy = 'manage_image_cache' try: api_policy.CacheImageAPIPolicy( - req.context, enforcer=self.policy).manage_image_cache() + req.context, image=image, enforcer=self.policy, + policy_str=new_policy).manage_image_cache() except exception.Forbidden: - LOG.debug("User not permitted to manage the image cache") + LOG.debug("User not permitted by '%s' policy" % new_policy) raise webob.exc.HTTPForbidden() + if not CONF.image_cache_dir: + msg = _("Caching via API is not supported at this site.") + raise webob.exc.HTTPNotFound(explanation=msg) + def get_cached_images(self, req): """ GET /cached_images @@ -114,6 +139,99 @@ class CacheController(object): self._enforce(req) return dict(num_deleted=self.cache.delete_all_queued_images()) + def delete_cache_entry(self, req, image_id): + """ + DELETE /cache/ - Remove image from cache + + Removes the image from cache or queue. + """ + image_repo = self.gateway.get_repo( + req.context, authorization_layer=False) + try: + image = image_repo.get(image_id) + except exception.NotFound: + # We are going to raise this error only if image is + # not present in cache or queue list + image = None + if not self.image_exists_in_cache(image_id): + msg = _("Image %s not found.") % image_id + LOG.warning(msg) + raise webob.exc.HTTPNotFound(explanation=msg) + + self._enforce(req, new_policy='cache_delete', image=image) + self.cache.delete_cached_image(image_id) + self.cache.delete_queued_image(image_id) + + def image_exists_in_cache(self, image_id): + queued_images = self.cache.get_queued_images() + if image_id in queued_images: + return True + + cached_images = self.cache.get_cached_images() + if image_id in [image['image_id'] for image in cached_images]: + return True + + return False + + def clear_cache(self, req): + """ + DELETE /cache - Clear cache and queue + + Removes all images from cache and queue. + """ + self._enforce(req, new_policy='cache_delete') + target = req.headers.get('x-image-cache-clear-target', '').lower() + if target == '': + res = dict(cache_deleted=self.cache.delete_all_cached_images(), + queue_deleted=self.cache.delete_all_queued_images()) + elif target == 'cache': + res = dict(cache_deleted=self.cache.delete_all_cached_images()) + elif target == 'queue': + res = dict(queue_deleted=self.cache.delete_all_queued_images()) + else: + reason = (_("If provided 'x-image-cache-clear-target' must be " + "'cache', 'queue' or empty string.")) + raise webob.exc.HTTPBadRequest(explanation=reason, + request=req, + content_type='text/plain') + return res + + def get_cache_state(self, req): + """ + GET /cache/ - Get currently cached and queued images + + Returns dict of cached and queued images + """ + self._enforce(req, new_policy='cache_list') + return dict(cached_images=self.cache.get_cached_images(), + queued_images=self.cache.get_queued_images()) + + def queue_image_from_api(self, req, image_id): + """ + PUT /cache/ + + Queues an image for caching. We do not check to see if + the image is in the registry here. That is done by the + prefetcher... + """ + image_repo = self.gateway.get_repo( + req.context, authorization_layer=False) + try: + image = image_repo.get(image_id) + except exception.NotFound: + msg = _("Image %s not found.") % image_id + LOG.warning(msg) + raise webob.exc.HTTPNotFound(explanation=msg) + + self._enforce(req, new_policy='cache_image', image=image) + + if image.status != 'active': + msg = _("Only images with status active can be targeted for " + "queueing") + raise webob.exc.HTTPBadRequest(explanation=msg) + + self.cache.queue_image(image_id) + class CachedImageDeserializer(wsgi.JSONRequestDeserializer): pass diff --git a/glance/api/v2/policy.py b/glance/api/v2/policy.py index 78f7a23d94..9edf55329b 100644 --- a/glance/api/v2/policy.py +++ b/glance/api/v2/policy.py @@ -105,14 +105,21 @@ class APIPolicyBase(object): class CacheImageAPIPolicy(APIPolicyBase): - def __init__(self, context, target=None, enforcer=None): + def __init__(self, context, image=None, policy_str=None, + target=None, enforcer=None): self._context = context - self._target = target or {} + target = {} + self._image = image + if self._image: + target = policy.ImageTarget(self._image) + + self._target = target self.enforcer = enforcer or policy.Enforcer() + self.policy_str = policy_str super(CacheImageAPIPolicy, self).__init__(context, target, enforcer) def manage_image_cache(self): - self._enforce('manage_image_cache') + self._enforce(self.policy_str) class ImageAPIPolicy(APIPolicyBase): diff --git a/glance/api/v2/router.py b/glance/api/v2/router.py index d3f115466f..d7e881af2a 100644 --- a/glance/api/v2/router.py +++ b/glance/api/v2/router.py @@ -13,6 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. +from glance.api.v2 import cached_images from glance.api.v2 import discovery from glance.api.v2 import image_actions from glance.api.v2 import image_data @@ -593,4 +594,32 @@ class API(wsgi.Router): action='get_usage', conditions={'method': ['GET']}) + # Cache Management API + cache_manage_resource = cached_images.create_resource() + mapper.connect('/cache', + controller=cache_manage_resource, + action='get_cache_state', + conditions={'method': ['GET']}, + body_reject=True) + mapper.connect('/cache', + controller=cache_manage_resource, + action='clear_cache', + conditions={'method': ['DELETE']}) + mapper.connect('/cache', + controller=reject_method_resource, + action='reject', + allowed_methods='GET, DELETE') + mapper.connect('/cache/{image_id}', + controller=cache_manage_resource, + action='delete_cache_entry', + conditions={'method': ['DELETE']}) + mapper.connect('/cache/{image_id}', + controller=cache_manage_resource, + action='queue_image_from_api', + conditions={'method': ['PUT']}) + mapper.connect('/cache/{image_id}', + controller=reject_method_resource, + action='reject', + allowed_methods='DELETE, PUT') + super(API, self).__init__(mapper) diff --git a/glance/api/versions.py b/glance/api/versions.py index f548e8a45f..18f77e3191 100644 --- a/glance/api/versions.py +++ b/glance/api/versions.py @@ -77,6 +77,15 @@ class Controller(object): } version_objs = [] + if CONF.image_cache_dir: + version_objs.extend([ + build_version_object(2.14, 'v2', 'CURRENT'), + build_version_object(2.13, 'v2', 'SUPPORTED'), + ]) + else: + version_objs.extend([ + build_version_object(2.13, 'v2', 'CURRENT'), + ]) if CONF.enabled_backends: version_objs.extend([ build_version_object(2.12, 'v2', 'SUPPORTED'), @@ -90,7 +99,6 @@ class Controller(object): build_version_object(2.9, 'v2', 'SUPPORTED'), ]) version_objs.extend([ - build_version_object(2.13, 'v2', 'CURRENT'), build_version_object(2.7, 'v2', 'SUPPORTED'), build_version_object(2.6, 'v2', 'SUPPORTED'), build_version_object(2.5, 'v2', 'SUPPORTED'), diff --git a/glance/policies/__init__.py b/glance/policies/__init__.py index 67b9dfc07b..a885bb3fc4 100644 --- a/glance/policies/__init__.py +++ b/glance/policies/__init__.py @@ -13,6 +13,7 @@ import itertools from glance.policies import base +from glance.policies import cache from glance.policies import image from glance.policies import metadef from glance.policies import tasks @@ -24,4 +25,5 @@ def list_rules(): image.list_rules(), tasks.list_rules(), metadef.list_rules(), + cache.list_rules(), ) diff --git a/glance/policies/base.py b/glance/policies/base.py index 3978966148..ef908eae27 100644 --- a/glance/policies/base.py +++ b/glance/policies/base.py @@ -90,6 +90,7 @@ ADMIN_OR_PROJECT_READER_OR_SHARED_MEMBER = ( f'role:reader and (project_id:%(project_id)s or {IMAGE_MEMBER_CHECK})' ) +ADMIN = f'role:admin' rules = [ policy.RuleDefault(name='default', check_str='', diff --git a/glance/policies/cache.py b/glance/policies/cache.py new file mode 100644 index 0000000000..ec8c1ccb1f --- /dev/null +++ b/glance/policies/cache.py @@ -0,0 +1,75 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from oslo_log import versionutils +from oslo_policy import policy + +from glance.policies import base + + +DEPRECATED_REASON = """ +The image API now supports roles. +""" + + +cache_policies = [ + policy.DocumentedRuleDefault( + name="cache_image", + check_str=base.ADMIN, + scope_types=['project'], + description='Queue image for caching', + operations=[ + {'path': '/v2/cache/{image_id}', + 'method': 'PUT'} + ], + deprecated_rule=policy.DeprecatedRule( + name="cache_image", check_str="rule:manage_image_cache", + deprecated_reason=DEPRECATED_REASON, + deprecated_since=versionutils.deprecated.XENA + ), + ), + policy.DocumentedRuleDefault( + name="cache_list", + check_str=base.ADMIN, + scope_types=['project'], + description='List cache status', + operations=[ + {'path': '/v2/cache', + 'method': 'GET'} + ], + deprecated_rule=policy.DeprecatedRule( + name="cache_list", check_str="rule:manage_image_cache", + deprecated_reason=DEPRECATED_REASON, + deprecated_since=versionutils.deprecated.XENA + ), + ), + policy.DocumentedRuleDefault( + name="cache_delete", + check_str=base.ADMIN, + scope_types=['project'], + description='Delete image(s) from cache and/or queue', + operations=[ + {'path': '/v2/cache', + 'method': 'DELETE'}, + {'path': '/v2/cache/{image_id}', + 'method': 'DELETE'} + ], + deprecated_rule=policy.DeprecatedRule( + name="cache_delete", check_str="rule:manage_image_cache", + deprecated_reason=DEPRECATED_REASON, + deprecated_since=versionutils.deprecated.XENA + ), + ), +] + + +def list_rules(): + return cache_policies diff --git a/glance/tests/functional/__init__.py b/glance/tests/functional/__init__.py index 14dab2d6c0..a0e03af393 100644 --- a/glance/tests/functional/__init__.py +++ b/glance/tests/functional/__init__.py @@ -1550,6 +1550,8 @@ class SynchronousAPIBase(test_utils.BaseTestCase): CacheManageFilter.factory [pipeline:glance-api-cachemanagement] pipeline = context cache cachemanage rootapp + [pipeline:glance-api-caching] + pipeline = context cache rootapp [pipeline:glance-api] pipeline = context rootapp [composite:rootapp] diff --git a/glance/tests/functional/test_api.py b/glance/tests/functional/test_api.py index fa5608d64e..cf93517a81 100644 --- a/glance/tests/functional/test_api.py +++ b/glance/tests/functional/test_api.py @@ -30,7 +30,8 @@ class TestApiVersions(functional.FunctionalTest): self.start_servers(**self.__dict__.copy()) url = 'http://127.0.0.1:%d' % self.api_port - versions = {'versions': tv.get_versions_list(url)} + versions = {'versions': tv.get_versions_list(url, + enabled_cache=True)} # Verify version choices returned. path = 'http://%s:%d' % ('127.0.0.1', self.api_port) @@ -44,7 +45,8 @@ class TestApiVersions(functional.FunctionalTest): self.start_servers(**self.__dict__.copy()) url = 'http://127.0.0.1:%d' % self.api_port - versions = {'versions': tv.get_versions_list(url)} + versions = {'versions': tv.get_versions_list(url, + enabled_cache=True)} # Verify version choices returned. path = 'http://%s:%d' % ('127.0.0.1', self.api_port) @@ -62,7 +64,8 @@ class TestApiVersionsMultistore(functional.MultipleBackendFunctionalTest): url = 'http://127.0.0.1:%d' % self.api_port versions = {'versions': tv.get_versions_list(url, - enabled_backends=True)} + enabled_backends=True, + enabled_cache=True)} # Verify version choices returned. path = 'http://%s:%d' % ('127.0.0.1', self.api_port) @@ -77,7 +80,8 @@ class TestApiVersionsMultistore(functional.MultipleBackendFunctionalTest): url = 'http://127.0.0.1:%d' % self.api_port versions = {'versions': tv.get_versions_list(url, - enabled_backends=True)} + enabled_backends=True, + enabled_cache=True)} # Verify version choices returned. path = 'http://%s:%d' % ('127.0.0.1', self.api_port) @@ -94,7 +98,8 @@ class TestApiPaths(functional.FunctionalTest): self.start_servers(**self.__dict__.copy()) url = 'http://127.0.0.1:%d' % self.api_port - self.versions = {'versions': tv.get_versions_list(url)} + self.versions = {'versions': tv.get_versions_list(url, + enabled_cache=True)} images = {'images': []} self.images_json = jsonutils.dumps(images) diff --git a/glance/tests/functional/v2/test_cache_api.py b/glance/tests/functional/v2/test_cache_api.py new file mode 100644 index 0000000000..bd3048aef5 --- /dev/null +++ b/glance/tests/functional/v2/test_cache_api.py @@ -0,0 +1,360 @@ +# Copyright 2021 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from unittest import mock + +import oslo_policy.policy + +from glance.api import policy +from glance.image_cache import prefetcher +from glance.tests import functional + + +class TestImageCache(functional.SynchronousAPIBase): + # ToDo(abhishekk): Once system scope is enabled and RBAC is fully + # supported, enable these tests for RBAC as well + def setUp(self): + super(TestImageCache, self).setUp() + self.policy = policy.Enforcer(suppress_deprecation_warnings=True) + + def set_policy_rules(self, rules): + self.policy.set_rules( + oslo_policy.policy.Rules.from_dict(rules), + overwrite=True) + + def start_server(self, enable_cache=True): + with mock.patch.object(policy, 'Enforcer') as mock_enf: + mock_enf.return_value = self.policy + super(TestImageCache, self).start_server(enable_cache=enable_cache) + + def load_data(self): + output = {} + # Create 1 queued image as well for testing + path = "/v2/images" + data = { + 'name': 'queued-image', + 'container_format': 'bare', + 'disk_format': 'raw' + } + response = self.api_post(path, json=data) + self.assertEqual(201, response.status_code) + image_id = response.json['id'] + output['queued'] = image_id + + for visibility in ['public', 'private', 'community', 'shared']: + data = { + 'name': '%s-image' % visibility, + 'visibility': visibility, + 'container_format': 'bare', + 'disk_format': 'raw' + } + response = self.api_post(path, json=data) + self.assertEqual(201, response.status_code) + image_id = response.json['id'] + # Upload some data to image + response = self.api_put( + '/v2/images/%s/file' % image_id, + headers={'Content-Type': 'application/octet-stream'}, + data=b'IMAGEDATA') + self.assertEqual(204, response.status_code) + output[visibility] = image_id + + return output + + def list_cache(self, expected_code=200): + path = '/v2/cache' + response = self.api_get(path) + self.assertEqual(expected_code, response.status_code) + if response.status_code == 200: + return response.json + + def cache_queue(self, image_id, expected_code=200): + # Queue image for prefetching + path = '/v2/cache/%s' % image_id + response = self.api_put(path) + self.assertEqual(expected_code, response.status_code) + + def cache_delete(self, image_id, expected_code=200): + path = '/v2/cache/%s' % image_id + response = self.api_delete(path) + self.assertEqual(expected_code, response.status_code) + + def cache_clear(self, target='', expected_code=200): + path = '/v2/cache' + headers = {} + if target: + headers['x-image-cache-clear-target'] = target + response = self.api_delete(path, headers=headers) + if target not in ('', 'cache', 'queue'): + self.assertEqual(expected_code, response.status_code) + else: + self.assertEqual(expected_code, response.status_code) + + def cache_image(self): + # NOTE(abhishekk): Here we are not running periodic job which caches + # queued images as precaching is not part of this patch, so to test + # all caching operations we are using this way to cache images for us + cache_prefetcher = prefetcher.Prefetcher() + cache_prefetcher.run() + + def test_cache_api_lifecycle(self): + self.start_server(enable_cache=True) + images = self.load_data() + + # Ensure that nothing is cached and nothing is queued for caching + output = self.list_cache() + self.assertEqual(0, len(output['queued_images'])) + self.assertEqual(0, len(output['cached_images'])) + + # Try non-existing image to queue for caching + self.cache_queue('non-existing-image-id', expected_code=404) + + # Verify that you can not queue non-active image + self.cache_queue(images['queued'], expected_code=400) + + # Queue 1 image for caching + self.cache_queue(images['public']) + # Now verify that we have 1 image queued for caching and 0 + # cached images + output = self.list_cache() + self.assertEqual(1, len(output['queued_images'])) + self.assertEqual(0, len(output['cached_images'])) + # Verify same image is queued for caching + self.assertIn(images['public'], output['queued_images']) + + # Cache the image + self.cache_image() + # Now verify that we have 0 queued image and 1 cached image + output = self.list_cache() + self.assertEqual(0, len(output['queued_images'])) + self.assertEqual(1, len(output['cached_images'])) + # Verify same image is queued for caching + self.assertIn(images['public'], output['cached_images'][0]['image_id']) + + # Queue 2nd image for caching + self.cache_queue(images['community']) + # Now verify that we have 1 image queued for caching and 1 + # cached images + output = self.list_cache() + self.assertEqual(1, len(output['queued_images'])) + self.assertEqual(1, len(output['cached_images'])) + # Verify same image is queued for caching + self.assertIn(images['community'], output['queued_images']) + self.assertIn(images['public'], output['cached_images'][0]['image_id']) + + # Queue 3rd image for caching + self.cache_queue(images['private']) + # Now verify that we have 2 images queued for caching and 1 + # cached images + output = self.list_cache() + self.assertEqual(2, len(output['queued_images'])) + self.assertEqual(1, len(output['cached_images'])) + # Verify same image is queued for caching + self.assertIn(images['private'], output['queued_images']) + + # Try to delete non-existing image from cache + self.cache_delete('non-existing-image-id', expected_code=404) + + # Delete public image from cache + self.cache_delete(images['public']) + # Now verify that we have 2 image queued for caching and no + # cached images + output = self.list_cache() + self.assertEqual(2, len(output['queued_images'])) + self.assertEqual(0, len(output['cached_images'])) + + # Verify clearing cache fails with 400 if invalid header is passed + self.cache_clear(target='both', expected_code=400) + + # Delete all queued images + self.cache_clear(target='queue') + # Now verify that we have 0 image queued for caching and 0 + # cached images + output = self.list_cache() + self.assertEqual(0, len(output['queued_images'])) + self.assertEqual(0, len(output['cached_images'])) + + # Queue and cache image so we have something to clear + self.cache_queue(images['public']) + # Now verify that we have 1 queued image + output = self.list_cache() + self.assertEqual(1, len(output['queued_images'])) + self.cache_image() + # Now verify that we have 0 queued image and 1 cached image + output = self.list_cache() + self.assertEqual(0, len(output['queued_images'])) + self.assertEqual(1, len(output['cached_images'])) + + # Delete all cached images + self.cache_clear(target='cache') + # Now verify that we have 0 image queued for caching and 0 + # cached images + output = self.list_cache() + self.assertEqual(0, len(output['queued_images'])) + self.assertEqual(0, len(output['cached_images'])) + + # Now we need 2 queued images and 2 cached images in order + # to delete both of them together + self.cache_queue(images['public']) + self.cache_queue(images['private']) + # Now verify that we have 2 queued images + output = self.list_cache() + self.assertEqual(2, len(output['queued_images'])) + + self.cache_image() + # Now verify that we have 0 queued images and 2 cached images + output = self.list_cache() + self.assertEqual(0, len(output['queued_images'])) + self.assertEqual(2, len(output['cached_images'])) + + self.cache_queue(images['community']) + self.cache_queue(images['shared']) + # Verify we have 2 queued and 2 cached images + output = self.list_cache() + self.assertEqual(2, len(output['queued_images'])) + self.assertEqual(2, len(output['cached_images'])) + + # Now delete all queued and all cached images at once + self.cache_clear() + # Now verify that we have 0 image queued for caching and 0 + # cached images + output = self.list_cache() + self.assertEqual(0, len(output['queued_images'])) + self.assertEqual(0, len(output['cached_images'])) + + # Try to cache image again to validate nothing will be cached + self.cache_image() + output = self.list_cache() + self.assertEqual(0, len(output['queued_images'])) + self.assertEqual(0, len(output['cached_images'])) + + def test_cache_image_queue_delete(self): + # This test verifies that if image is queued for caching + # and user deletes the original image, but it is still + # present in queued list and deleted with cache-delete API. + self.start_server(enable_cache=True) + images = self.load_data() + + # Ensure that nothing is cached and nothing is queued for caching + output = self.list_cache() + self.assertEqual(0, len(output['queued_images'])) + self.assertEqual(0, len(output['cached_images'])) + + self.cache_queue(images['public']) + # Now verify that we have 1 image queued for caching and 0 + # cached images + output = self.list_cache() + self.assertEqual(1, len(output['queued_images'])) + self.assertEqual(0, len(output['cached_images'])) + # Verify same image is queued for caching + self.assertIn(images['public'], output['queued_images']) + + # Delete image and verify that it is still present + # in queued list + path = '/v2/images/%s' % images['public'] + response = self.api_delete(path) + self.assertEqual(204, response.status_code) + + output = self.list_cache() + self.assertEqual(1, len(output['queued_images'])) + self.assertEqual(0, len(output['cached_images'])) + self.assertIn(images['public'], output['queued_images']) + + # Deleted the image from queued list + self.cache_delete(images['public']) + output = self.list_cache() + self.assertEqual(0, len(output['queued_images'])) + self.assertEqual(0, len(output['cached_images'])) + + def test_cache_image_cache_delete(self): + # This test verifies that if image is queued for caching + # and user deletes the original image, but it is still + # present in queued list and deleted with cache-delete API. + self.start_server(enable_cache=True) + images = self.load_data() + + # Ensure that nothing is cached and nothing is queued for caching + output = self.list_cache() + self.assertEqual(0, len(output['queued_images'])) + self.assertEqual(0, len(output['cached_images'])) + + self.cache_queue(images['public']) + # Now verify that we have 1 image queued for caching and 0 + # cached images + output = self.list_cache() + self.assertEqual(1, len(output['queued_images'])) + self.assertEqual(0, len(output['cached_images'])) + # Verify same image is queued for caching + self.assertIn(images['public'], output['queued_images']) + + # Cache the image + self.cache_image() + # Now verify that we have 0 queued image and 1 cached image + output = self.list_cache() + self.assertEqual(0, len(output['queued_images'])) + self.assertEqual(1, len(output['cached_images'])) + # Verify same image is queued for caching + self.assertIn(images['public'], output['cached_images'][0]['image_id']) + + # Delete image and verify that it is deleted from + # cache as well + path = '/v2/images/%s' % images['public'] + response = self.api_delete(path) + self.assertEqual(204, response.status_code) + + output = self.list_cache() + self.assertEqual(0, len(output['queued_images'])) + self.assertEqual(0, len(output['cached_images'])) + + def test_cache_api_cache_disabled(self): + self.start_server(enable_cache=False) + images = self.load_data() + # As cache is not enabled each API call should return 404 response + self.list_cache(expected_code=404) + self.cache_queue(images['public'], expected_code=404) + self.cache_delete(images['public'], expected_code=404) + self.cache_clear(expected_code=404) + self.cache_clear(target='both', expected_code=404) + + # Now disable cache policies and ensure that you will get 403 + self.set_policy_rules({ + 'cache_list': '!', + 'cache_delete': '!', + 'cache_image': '!', + 'add_image': '', + 'upload_image': '' + }) + self.list_cache(expected_code=403) + self.cache_queue(images['public'], expected_code=403) + self.cache_delete(images['public'], expected_code=403) + self.cache_clear(expected_code=403) + self.cache_clear(target='both', expected_code=403) + + def test_cache_api_not_allowed(self): + self.start_server(enable_cache=True) + images = self.load_data() + # As cache operations are not allowed each API call should return + # 403 response + self.set_policy_rules({ + 'cache_list': '!', + 'cache_delete': '!', + 'cache_image': '!', + 'add_image': '', + 'upload_image': '' + }) + self.list_cache(expected_code=403) + self.cache_queue(images['public'], expected_code=403) + self.cache_delete(images['public'], expected_code=403) + self.cache_clear(expected_code=403) + self.cache_clear(target='both', expected_code=403) diff --git a/glance/tests/unit/test_cached_images.py b/glance/tests/unit/test_cached_images.py index 4617bbdaaa..5be3df5aba 100644 --- a/glance/tests/unit/test_cached_images.py +++ b/glance/tests/unit/test_cached_images.py @@ -13,30 +13,51 @@ # License for the specific language governing permissions and limitations # under the License. -import testtools +from unittest import mock + import webob -from glance.api import policy from glance.api.v2 import cached_images -from glance.common import exception +import glance.gateway from glance import image_cache +from glance import notifier +import glance.tests.unit.utils as unit_test_utils +import glance.tests.utils as test_utils -class FakePolicyEnforcer(policy.Enforcer): - def __init__(self): - self.default_rule = '' - self.policy_path = '' - self.policy_file_mtime = None - self.policy_file_contents = None +UUID4 = '6bbe7cc2-eae7-4c0f-b50d-a7160b0c6a86' - def enforce(self, context, action, target): - return 'pass' - def check(rule, target, creds, exc=None, *args, **kwargs): - return 'pass' +class FakeImage(object): + def __init__(self, id=None, status='active', container_format='ami', + disk_format='ami', locations=None): + self.id = id or UUID4 + self.status = status + self.container_format = container_format + self.disk_format = disk_format + self.locations = locations + self.owner = unit_test_utils.TENANT1 + self.created_at = '' + self.updated_at = '' + self.min_disk = '' + self.min_ram = '' + self.protected = False + self.checksum = '' + self.os_hash_algo = '' + self.os_hash_value = '' + self.size = 0 + self.virtual_size = 0 + self.visibility = 'public' + self.os_hidden = False + self.name = 'foo' + self.tags = [] + self.extra_properties = {} + self.member = self.owner - def _check(self, context, rule, target, *args, **kwargs): - return 'pass' + # NOTE(danms): This fixture looks more like the db object than + # the proxy model. This needs fixing all through the tests + # below. + self.image_id = self.id class FakeCache(image_cache.ImageCache): @@ -48,13 +69,14 @@ class FakeCache(image_cache.ImageCache): pass def get_cached_images(self): - return {'id': 'test'} + return [{'image_id': 'test'}] def delete_cached_image(self, image_id): self.deleted_images.append(image_id) def delete_all_cached_images(self): - self.delete_cached_image(self.get_cached_images().get('id')) + self.delete_cached_image( + self.get_cached_images()[0].get('image_id')) return 1 def get_queued_images(self): @@ -74,72 +96,315 @@ class FakeCache(image_cache.ImageCache): class FakeController(cached_images.CacheController): def __init__(self): self.cache = FakeCache() - self.policy = FakePolicyEnforcer() + self.db = unit_test_utils.FakeDB(initialize=False) + self.policy = unit_test_utils.FakePolicyEnforcer() + self.notifier = unit_test_utils.FakeNotifier() + self.store = unit_test_utils.FakeStoreAPI() + self.gateway = glance.gateway.Gateway(self.db, self.store, + self.notifier, self.policy) -class TestController(testtools.TestCase): +class TestController(test_utils.BaseTestCase): def test_initialization_without_conf(self): - self.assertRaises(exception.BadDriverConfiguration, - cached_images.CacheController) + # NOTE(abhishekk): Since we are initializing cache driver only + # if image_cache_dir is set, here we are checking that cache + # object is None when it is not set + caching_controller = cached_images.CacheController() + self.assertIsNone(caching_controller.cache) -class TestCachedImages(testtools.TestCase): +class TestCachedImages(test_utils.BaseTestCase): def setUp(self): super(TestCachedImages, self).setUp() test_controller = FakeController() self.controller = test_controller def test_get_cached_images(self): + self.config(image_cache_dir='fake_cache_directory') req = webob.Request.blank('') req.context = 'test' result = self.controller.get_cached_images(req) - self.assertEqual({'cached_images': {'id': 'test'}}, result) + self.assertEqual({'cached_images': [{'image_id': 'test'}]}, result) def test_delete_cached_image(self): - req = webob.Request.blank('') - req.context = 'test' - self.controller.delete_cached_image(req, image_id='test') - self.assertEqual(['test'], self.controller.cache.deleted_images) + self.config(image_cache_dir='fake_cache_directory') + req = unit_test_utils.get_fake_request() + with mock.patch.object(notifier.ImageRepoProxy, + 'get') as mock_get: + mock_get.return_value = FakeImage() + self.controller.delete_cached_image(req, image_id=UUID4) + self.assertEqual([UUID4], self.controller.cache.deleted_images) def test_delete_cached_images(self): + self.config(image_cache_dir='fake_cache_directory') req = webob.Request.blank('') req.context = 'test' self.assertEqual({'num_deleted': 1}, self.controller.delete_cached_images(req)) self.assertEqual(['test'], self.controller.cache.deleted_images) - def test_policy_enforce_forbidden(self): - def fake_enforce(context, action, target): - raise exception.Forbidden() - - self.controller.policy.enforce = fake_enforce - req = webob.Request.blank('') - req.context = 'test' - self.assertRaises(webob.exc.HTTPForbidden, - self.controller.get_cached_images, req) - def test_get_queued_images(self): + self.config(image_cache_dir='fake_cache_directory') req = webob.Request.blank('') req.context = 'test' result = self.controller.get_queued_images(req) self.assertEqual({'queued_images': {'test': 'passed'}}, result) def test_queue_image(self): - req = webob.Request.blank('') - req.context = 'test' - self.controller.queue_image(req, image_id='test1') + self.config(image_cache_dir='fake_cache_directory') + req = unit_test_utils.get_fake_request() + with mock.patch.object(notifier.ImageRepoProxy, + 'get') as mock_get: + mock_get.return_value = FakeImage() + self.controller.queue_image(req, image_id=UUID4) def test_delete_queued_image(self): - req = webob.Request.blank('') - req.context = 'test' - self.controller.delete_queued_image(req, 'deleted_img') - self.assertEqual(['deleted_img'], - self.controller.cache.deleted_images) + self.config(image_cache_dir='fake_cache_directory') + req = unit_test_utils.get_fake_request() + with mock.patch.object(notifier.ImageRepoProxy, + 'get') as mock_get: + mock_get.return_value = FakeImage() + self.controller.delete_queued_image(req, UUID4) + self.assertEqual([UUID4], + self.controller.cache.deleted_images) def test_delete_queued_images(self): + self.config(image_cache_dir='fake_cache_directory') req = webob.Request.blank('') req.context = 'test' self.assertEqual({'num_deleted': 1}, self.controller.delete_queued_images(req)) self.assertEqual(['deleted_img'], self.controller.cache.deleted_images) + + +class TestCachedImagesNegative(test_utils.BaseTestCase): + def setUp(self): + super(TestCachedImagesNegative, self).setUp() + test_controller = FakeController() + self.controller = test_controller + + def test_get_cached_images_disabled(self): + req = webob.Request.blank('') + req.context = 'test' + self.assertRaises(webob.exc.HTTPNotFound, + self.controller.get_cached_images, req) + + def test_get_cached_images_forbidden(self): + self.config(image_cache_dir='fake_cache_directory') + self.controller.policy.rules = {"manage_image_cache": False} + req = unit_test_utils.get_fake_request() + with mock.patch.object(notifier.ImageRepoProxy, + 'get') as mock_get: + mock_get.return_value = FakeImage() + self.assertRaises(webob.exc.HTTPForbidden, + self.controller.get_cached_images, + req) + + def test_delete_cached_image_disabled(self): + req = webob.Request.blank('') + req.context = 'test' + self.assertRaises(webob.exc.HTTPNotFound, + self.controller.delete_cached_image, req, + image_id='test') + + def test_delete_cached_image_forbidden(self): + self.config(image_cache_dir='fake_cache_directory') + self.controller.policy.rules = {"manage_image_cache": False} + req = unit_test_utils.get_fake_request() + with mock.patch.object(notifier.ImageRepoProxy, + 'get') as mock_get: + mock_get.return_value = FakeImage() + self.assertRaises(webob.exc.HTTPForbidden, + self.controller.delete_cached_image, + req, image_id=UUID4) + + def test_delete_cached_images_disabled(self): + req = webob.Request.blank('') + req.context = 'test' + self.assertRaises(webob.exc.HTTPNotFound, + self.controller.delete_cached_images, req) + + def test_delete_cached_images_forbidden(self): + self.config(image_cache_dir='fake_cache_directory') + self.controller.policy.rules = {"manage_image_cache": False} + req = unit_test_utils.get_fake_request() + with mock.patch.object(notifier.ImageRepoProxy, + 'get') as mock_get: + mock_get.return_value = FakeImage() + self.assertRaises(webob.exc.HTTPForbidden, + self.controller.delete_cached_images, + req) + + def test_get_queued_images_disabled(self): + req = webob.Request.blank('') + req.context = 'test' + self.assertRaises(webob.exc.HTTPNotFound, + self.controller.get_queued_images, req) + + def test_get_queued_images_forbidden(self): + self.config(image_cache_dir='fake_cache_directory') + self.controller.policy.rules = {"manage_image_cache": False} + req = unit_test_utils.get_fake_request() + with mock.patch.object(notifier.ImageRepoProxy, + 'get') as mock_get: + mock_get.return_value = FakeImage() + self.assertRaises(webob.exc.HTTPForbidden, + self.controller.get_queued_images, + req) + + def test_queue_image_disabled(self): + req = webob.Request.blank('') + req.context = 'test' + self.assertRaises(webob.exc.HTTPNotFound, + self.controller.queue_image, + req, image_id='test1') + + def test_queue_image_forbidden(self): + self.config(image_cache_dir='fake_cache_directory') + self.controller.policy.rules = {"manage_image_cache": False} + req = unit_test_utils.get_fake_request() + with mock.patch.object(notifier.ImageRepoProxy, + 'get') as mock_get: + mock_get.return_value = FakeImage() + self.assertRaises(webob.exc.HTTPForbidden, + self.controller.queue_image, + req, image_id=UUID4) + + def test_delete_queued_image_disabled(self): + req = webob.Request.blank('') + req.context = 'test' + self.assertRaises(webob.exc.HTTPNotFound, + self.controller.delete_queued_image, + req, image_id='test1') + + def test_delete_queued_image_forbidden(self): + self.config(image_cache_dir='fake_cache_directory') + self.controller.policy.rules = {"manage_image_cache": False} + req = unit_test_utils.get_fake_request() + with mock.patch.object(notifier.ImageRepoProxy, + 'get') as mock_get: + mock_get.return_value = FakeImage() + self.assertRaises(webob.exc.HTTPForbidden, + self.controller.delete_queued_image, + req, image_id=UUID4) + + def test_delete_queued_images_disabled(self): + req = webob.Request.blank('') + req.context = 'test' + self.assertRaises(webob.exc.HTTPNotFound, + self.controller.delete_queued_images, req) + + def test_delete_queued_images_forbidden(self): + self.config(image_cache_dir='fake_cache_directory') + self.controller.policy.rules = {"manage_image_cache": False} + req = unit_test_utils.get_fake_request() + with mock.patch.object(notifier.ImageRepoProxy, + 'get') as mock_get: + mock_get.return_value = FakeImage() + self.assertRaises(webob.exc.HTTPForbidden, + self.controller.delete_queued_images, + req) + + def test_delete_cache_entry_forbidden(self): + self.config(image_cache_dir='fake_cache_directory') + self.controller.policy.rules = {"cache_delete": False} + req = unit_test_utils.get_fake_request() + with mock.patch.object(notifier.ImageRepoProxy, + 'get') as mock_get: + mock_get.return_value = FakeImage() + self.assertRaises(webob.exc.HTTPForbidden, + self.controller.delete_cache_entry, + req, image_id=UUID4) + + def test_delete_cache_entry_disabled(self): + req = unit_test_utils.get_fake_request() + with mock.patch.object(notifier.ImageRepoProxy, + 'get') as mock_get: + mock_get.return_value = FakeImage() + self.assertRaises(webob.exc.HTTPNotFound, + self.controller.delete_cache_entry, + req, image_id=UUID4) + + def test_delete_non_existing_cache_entries(self): + self.config(image_cache_dir='fake_cache_directory') + req = unit_test_utils.get_fake_request() + self.assertRaises(webob.exc.HTTPNotFound, + self.controller.delete_cache_entry, + req, image_id='non-existing-queued-image') + + def test_clear_cache_forbidden(self): + self.config(image_cache_dir='fake_cache_directory') + self.controller.policy.rules = {"cache_delete": False} + req = unit_test_utils.get_fake_request() + self.assertRaises(webob.exc.HTTPForbidden, + self.controller.clear_cache, + req) + + def test_clear_cache_disabled(self): + req = webob.Request.blank('') + req.context = 'test' + self.assertRaises(webob.exc.HTTPNotFound, + self.controller.clear_cache, req) + + def test_cache_clear_invalid_target(self): + self.config(image_cache_dir='fake_cache_directory') + req = unit_test_utils.get_fake_request() + req.headers.update({'x-image-cache-clear-target': 'invalid'}) + self.assertRaises(webob.exc.HTTPBadRequest, + self.controller.clear_cache, + req) + + def test_get_cache_state_disabled(self): + req = webob.Request.blank('') + req.context = 'test' + self.assertRaises(webob.exc.HTTPNotFound, + self.controller.get_cache_state, req) + + def test_get_cache_state_forbidden(self): + self.config(image_cache_dir='fake_cache_directory') + self.controller.policy.rules = {"cache_list": False} + req = unit_test_utils.get_fake_request() + with mock.patch.object(notifier.ImageRepoProxy, + 'get') as mock_get: + mock_get.return_value = FakeImage() + self.assertRaises(webob.exc.HTTPForbidden, + self.controller.get_cache_state, + req) + + def test_queue_image_from_api_disabled(self): + req = webob.Request.blank('') + req.context = 'test' + self.assertRaises(webob.exc.HTTPNotFound, + self.controller.queue_image_from_api, + req, image_id='test1') + + def test_queue_image_from_api_forbidden(self): + self.config(image_cache_dir='fake_cache_directory') + self.controller.policy.rules = {"cache_image": False} + req = unit_test_utils.get_fake_request() + with mock.patch.object(notifier.ImageRepoProxy, + 'get') as mock_get: + mock_get.return_value = FakeImage() + self.assertRaises(webob.exc.HTTPForbidden, + self.controller.queue_image_from_api, + req, image_id=UUID4) + + def test_non_active_image_for_queue_api(self): + self.config(image_cache_dir='fake_cache_directory') + req = unit_test_utils.get_fake_request() + for status in ('saving', 'queued', 'pending_delete', + 'deactivated', 'importing', 'uploading'): + with mock.patch.object(notifier.ImageRepoProxy, + 'get') as mock_get: + mock_get.return_value = FakeImage(status=status) + self.assertRaises(webob.exc.HTTPBadRequest, + self.controller.queue_image_from_api, + req, image_id=UUID4) + + def test_queue_api_non_existing_image_(self): + self.config(image_cache_dir='fake_cache_directory') + req = unit_test_utils.get_fake_request() + self.assertRaises(webob.exc.HTTPNotFound, + self.controller.queue_image_from_api, + req, image_id='non-existing-image-id') diff --git a/glance/tests/unit/test_versions.py b/glance/tests/unit/test_versions.py index aeeeda4083..b33b3dbc3d 100644 --- a/glance/tests/unit/test_versions.py +++ b/glance/tests/unit/test_versions.py @@ -28,7 +28,8 @@ from glance.tests.unit import base # make this public so it doesn't need to be repeated for the # functional tests -def get_versions_list(url, enabled_backends=False): +def get_versions_list(url, enabled_backends=False, + enabled_cache=False): image_versions = [ { 'id': 'v2.13', @@ -36,6 +37,12 @@ def get_versions_list(url, enabled_backends=False): 'links': [{'rel': 'self', 'href': '%s/v2/' % url}], }, + { + 'id': 'v2.9', + 'status': 'SUPPORTED', + 'links': [{'rel': 'self', + 'href': '%s/v2/' % url}], + }, { 'id': 'v2.7', 'status': 'SUPPORTED', @@ -87,6 +94,12 @@ def get_versions_list(url, enabled_backends=False): ] if enabled_backends: image_versions = [ + { + 'id': 'v2.13', + 'status': 'CURRENT', + 'links': [{'rel': 'self', + 'href': '%s/v2/' % url}], + }, { 'id': 'v2.12', 'status': 'SUPPORTED', @@ -117,14 +130,16 @@ def get_versions_list(url, enabled_backends=False): 'links': [{'rel': 'self', 'href': '%s/v2/' % url}], } - ] + image_versions - else: + ] + image_versions[2:] + + if enabled_cache: image_versions.insert(0, { - 'id': 'v2.9', - 'status': 'SUPPORTED', + 'id': 'v2.14', + 'status': 'CURRENT', 'links': [{'rel': 'self', 'href': '%s/v2/' % url}], }) + image_versions[1]['status'] = 'SUPPORTED' return image_versions @@ -151,6 +166,14 @@ class VersionsTest(base.IsolatedUnitTest): enabled_backends=True) self.assertEqual(expected, results) + self.config(image_cache_dir='/tmp/cache') + res = versions.Controller().index(req) + results = jsonutils.loads(res.body)['versions'] + expected = get_versions_list('http://127.0.0.1:9292', + enabled_backends=True, + enabled_cache=True) + self.assertEqual(expected, results) + def test_get_version_list_public_endpoint(self): req = webob.Request.blank('/', base_url='http://127.0.0.1:9292/') req.accept = 'application/json' @@ -170,6 +193,14 @@ class VersionsTest(base.IsolatedUnitTest): enabled_backends=True) self.assertEqual(expected, results) + self.config(image_cache_dir='/tmp/cache') + res = versions.Controller().index(req) + results = jsonutils.loads(res.body)['versions'] + expected = get_versions_list('https://example.com:9292', + enabled_backends=True, + enabled_cache=True) + self.assertEqual(expected, results) + def test_get_version_list_secure_proxy_ssl_header(self): self.config(secure_proxy_ssl_header='HTTP_X_FORWARDED_PROTO') url = 'http://localhost:9292' @@ -188,6 +219,14 @@ class VersionsTest(base.IsolatedUnitTest): expected = get_versions_list(url, enabled_backends=True) self.assertEqual(expected, results) + self.config(image_cache_dir='/tmp/cache') + res = versions.Controller().index(req) + results = jsonutils.loads(res.body)['versions'] + expected = get_versions_list(url, + enabled_backends=True, + enabled_cache=True) + self.assertEqual(expected, results) + def test_get_version_list_secure_proxy_ssl_header_https(self): self.config(secure_proxy_ssl_header='HTTP_X_FORWARDED_PROTO') url = 'http://localhost:9292' @@ -208,6 +247,14 @@ class VersionsTest(base.IsolatedUnitTest): expected = get_versions_list(ssl_url, enabled_backends=True) self.assertEqual(expected, results) + self.config(image_cache_dir='/tmp/cache') + res = versions.Controller().index(req) + results = jsonutils.loads(res.body)['versions'] + expected = get_versions_list(ssl_url, + enabled_backends=True, + enabled_cache=True) + self.assertEqual(expected, results) + def test_get_version_list_for_external_app(self): url = 'http://customhost:9292/app/api' req = webob.Request.blank('/', base_url=url) @@ -225,6 +272,13 @@ class VersionsTest(base.IsolatedUnitTest): expected = get_versions_list(url, enabled_backends=True) self.assertEqual(expected, results) + self.config(image_cache_dir='/tmp/cache') + res = versions.Controller().index(req) + results = jsonutils.loads(res.body)['versions'] + expected = get_versions_list(url, + enabled_backends=True, + enabled_cache=True) + class VersionNegotiationTest(base.IsolatedUnitTest): @@ -333,15 +387,21 @@ class VersionNegotiationTest(base.IsolatedUnitTest): self.middleware.process_request(request) self.assertEqual('/v2/images', request.path_info) - # version 2.14 does not exist - def test_request_url_v2_14_default_unsupported(self): + def test_request_url_v2_14_enabled_supported(self): + self.config(image_cache_dir='/tmp/cache') request = webob.Request.blank('/v2.14/images') + self.middleware.process_request(request) + self.assertEqual('/v2/images', request.path_info) + + # version 2.15 does not exist + def test_request_url_v2_15_default_unsupported(self): + request = webob.Request.blank('/v2.15/images') resp = self.middleware.process_request(request) self.assertIsInstance(resp, versions.Controller) - def test_request_url_v2_14_enabled_unsupported(self): - self.config(enabled_backends='slow:one,fast:two') - request = webob.Request.blank('/v2.14/images') + def test_request_url_v2_15_enabled_unsupported(self): + self.config(image_cache_dir='/tmp/cache') + request = webob.Request.blank('/v2.15/images') resp = self.middleware.process_request(request) self.assertIsInstance(resp, versions.Controller) diff --git a/glance/tests/unit/v2/test_cache_management_api.py b/glance/tests/unit/v2/test_cache_management_api.py new file mode 100644 index 0000000000..ea45f6848b --- /dev/null +++ b/glance/tests/unit/v2/test_cache_management_api.py @@ -0,0 +1,123 @@ +# Copyright 2021 Red Hat Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from unittest import mock + +from glance.api.v2 import cached_images +from glance import notifier +import glance.tests.unit.utils as unit_test_utils +import glance.tests.utils as test_utils + + +UUID1 = 'c80a1a6c-bd1f-41c5-90ee-81afedb1d58d' + + +class FakeImage(object): + def __init__(self, id=None, status='active', container_format='ami', + disk_format='ami', locations=None): + self.id = id or UUID1 + self.status = status + self.container_format = container_format + self.disk_format = disk_format + self.locations = locations + self.owner = unit_test_utils.TENANT1 + self.created_at = '' + self.updated_at = '' + self.min_disk = '' + self.min_ram = '' + self.protected = False + self.checksum = '' + self.os_hash_algo = '' + self.os_hash_value = '' + self.size = 0 + self.virtual_size = 0 + self.visibility = 'public' + self.os_hidden = False + self.name = 'foo' + self.tags = [] + self.extra_properties = {} + self.member = self.owner + + # NOTE(danms): This fixture looks more like the db object than + # the proxy model. This needs fixing all through the tests + # below. + self.image_id = self.id + + +class TestCacheManageAPI(test_utils.BaseTestCase): + + def setUp(self): + super(TestCacheManageAPI, self).setUp() + self.req = unit_test_utils.get_fake_request() + + def _main_test_helper(self, argv, status='active', image_mock=True): + with mock.patch.object(notifier.ImageRepoProxy, + 'get') as mock_get: + image = FakeImage(status=status) + mock_get.return_value = image + with mock.patch.object(cached_images.CacheController, + '_enforce') as e: + with mock.patch('glance.image_cache.ImageCache') as ic: + cc = cached_images.CacheController() + cc.cache = ic + c_calls = [] + c_calls += argv[0].split(',') + for call in c_calls: + mock.patch.object(ic, call) + test_call = getattr(cc, argv[1]) + new_policy = argv[2] + args = [] + if len(argv) == 4: + args = argv[3:] + test_call(self.req, *args) + if image_mock: + e.assert_called_once_with(self.req, image=image, + new_policy=new_policy) + else: + e.assert_called_once_with(self.req, + new_policy=new_policy) + mcs = [] + for method in ic.method_calls: + mcs.append(str(method)) + for call in c_calls: + if args == []: + args.append("") + elif args[0] and not args[0].endswith("'"): + args[0] = "'" + args[0] + "'" + self.assertIn("call." + call + "(" + args[0] + ")", + mcs) + self.assertEqual(len(c_calls), len(mcs)) + + def test_delete_cache_entry(self): + self._main_test_helper(['delete_cached_image,delete_queued_image', + 'delete_cache_entry', + 'cache_delete', + UUID1]) + + def test_clear_cache(self): + self._main_test_helper( + ['delete_all_cached_images,delete_all_queued_images', + 'clear_cache', + 'cache_delete'], image_mock=False) + + def test_get_cache_state(self): + self._main_test_helper(['get_cached_images,get_queued_images', + 'get_cache_state', + 'cache_list'], image_mock=False) + + def test_queue_image_from_api(self): + self._main_test_helper(['queue_image', + 'queue_image_from_api', + 'cache_image', + UUID1]) diff --git a/glance/tests/unit/v2/test_v2_policy.py b/glance/tests/unit/v2/test_v2_policy.py index 9bb296a550..0b2c2bb995 100644 --- a/glance/tests/unit/v2/test_v2_policy.py +++ b/glance/tests/unit/v2/test_v2_policy.py @@ -780,16 +780,44 @@ class TestTasksAPIPolicy(APIPolicyBase): mock.ANY) -class TestCacheImageAPIPolicy(APIPolicyBase): +class TestCacheImageAPIPolicy(utils.BaseTestCase): def setUp(self): super(TestCacheImageAPIPolicy, self).setUp() self.enforcer = mock.MagicMock() self.context = mock.MagicMock() - self.policy = policy.CacheImageAPIPolicy( - self.context, enforcer=self.enforcer) def test_manage_image_cache(self): + self.policy = policy.CacheImageAPIPolicy( + self.context, enforcer=self.enforcer, + policy_str='manage_image_cache') self.policy.manage_image_cache() self.enforcer.enforce.assert_called_once_with(self.context, 'manage_image_cache', mock.ANY) + + def test_manage_image_cache_with_cache_delete(self): + self.policy = policy.CacheImageAPIPolicy( + self.context, enforcer=self.enforcer, + policy_str='cache_delete') + self.policy.manage_image_cache() + self.enforcer.enforce.assert_called_once_with(self.context, + 'cache_delete', + mock.ANY) + + def test_manage_image_cache_with_cache_list(self): + self.policy = policy.CacheImageAPIPolicy( + self.context, enforcer=self.enforcer, + policy_str='cache_list') + self.policy.manage_image_cache() + self.enforcer.enforce.assert_called_once_with(self.context, + 'cache_list', + mock.ANY) + + def test_manage_image_cache_with_cache_image(self): + self.policy = policy.CacheImageAPIPolicy( + self.context, enforcer=self.enforcer, + policy_str='cache_image') + self.policy.manage_image_cache() + self.enforcer.enforce.assert_called_once_with(self.context, + 'cache_image', + mock.ANY) diff --git a/releasenotes/notes/cache-api-b806ccfb8c5d9bb6.yaml b/releasenotes/notes/cache-api-b806ccfb8c5d9bb6.yaml new file mode 100644 index 0000000000..d061fb9f13 --- /dev/null +++ b/releasenotes/notes/cache-api-b806ccfb8c5d9bb6.yaml @@ -0,0 +1,9 @@ +--- +features: + - | + This release introduces new APIs for cache related operations. This new + version of the cache API will help administrators to cache images on + dedicated glance nodes as well. For more information, see the + ``Cache Manage`` section in the `api-ref-guide + `_. +