From 514258619e5eb145e0c41d1341276ffa7ce478ff Mon Sep 17 00:00:00 2001 From: Zhi Yan Liu Date: Sun, 27 Oct 2013 14:33:53 +0800 Subject: [PATCH] Apply image location selection strategy Apply image location selection strategy into Glance server side. Image download handling and "direct URL" exporting will be effected by this mechanism. Implements bp: image-location-selection-strategy Related-Id: I86f192aeae8e5f21a72f946552f6507654c25a6c Change-Id: I7bd093a16db3af2b604cad22a6b6971345af82a2 Signed-off-by: Zhi Yan Liu --- glance/api/policy.py | 9 ++ glance/api/v2/images.py | 6 +- glance/db/__init__.py | 3 +- glance/quota/__init__.py | 9 ++ glance/store/__init__.py | 10 ++ glance/tests/functional/__init__.py | 6 + glance/tests/functional/v2/test_images.py | 157 ++++++++++++++++++++++ 7 files changed, 198 insertions(+), 2 deletions(-) diff --git a/glance/api/policy.py b/glance/api/policy.py index bc804d519f..b8bccee01c 100644 --- a/glance/api/policy.py +++ b/glance/api/policy.py @@ -16,6 +16,7 @@ """Policy Engine For Glance""" +import copy import os.path from oslo.config import cfg @@ -304,6 +305,14 @@ class ImageLocationsProxy(object): self.context = context self.policy = policy + def __copy__(self): + return type(self)(self.locations, self.context, self.policy) + + def __deepcopy__(self, memo): + # NOTE(zhiyan): Only copy location entries, others can be reused. + return type(self)(copy.deepcopy(self.locations, memo), + self.context, self.policy) + def _get_checker(action, func_name): def _checker(self, *args, **kwargs): self.policy.enforce(self.context, action, {}) diff --git a/glance/api/v2/images.py b/glance/api/v2/images.py index 600219c0a4..3e8d5108dc 100644 --- a/glance/api/v2/images.py +++ b/glance/api/v2/images.py @@ -21,6 +21,7 @@ import webob.exc from glance.api import policy from glance.common import exception +from glance.common import location_strategy from glance.common import utils from glance.common import wsgi import glance.db @@ -582,7 +583,10 @@ class ResponseSerializer(wsgi.JSONResponseSerializer): image_view['locations'] = [] if CONF.show_image_direct_url and image.locations: - image_view['direct_url'] = image.locations[0]['url'] + # Choose best location configured strategy + best_location = ( + location_strategy.choose_best_location(image.locations)) + image_view['direct_url'] = best_location['url'] image_view['tags'] = list(image.tags) image_view['self'] = self._get_image_href(image) diff --git a/glance/db/__init__.py b/glance/db/__init__.py index 63230cade0..6ea8cfac2b 100644 --- a/glance/db/__init__.py +++ b/glance/db/__init__.py @@ -20,6 +20,7 @@ from oslo.config import cfg from glance.common import crypt from glance.common import exception +from glance.common import location_strategy import glance.domain import glance.domain.proxy from glance.openstack.common import importutils @@ -105,7 +106,7 @@ class ImageRepo(object): min_disk=db_image['min_disk'], min_ram=db_image['min_ram'], protected=db_image['protected'], - locations=locations, + locations=location_strategy.get_ordered_locations(locations), checksum=db_image['checksum'], owner=db_image['owner'], disk_format=db_image['disk_format'], diff --git a/glance/quota/__init__.py b/glance/quota/__init__.py index 6f3b7eb217..7921d79488 100644 --- a/glance/quota/__init__.py +++ b/glance/quota/__init__.py @@ -12,6 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. +import copy from oslo.config import cfg @@ -242,6 +243,14 @@ class QuotaImageLocationsProxy(object): self.db_api) _enforce_image_location_quota(self.image, locations) + def __copy__(self): + return type(self)(self.image, self.context, self.db_api) + + def __deepcopy__(self, memo): + # NOTE(zhiyan): Only copy location entries, others can be reused. + self.image.locations = copy.deepcopy(self.locations, memo) + return type(self)(self.image, self.context, self.db_api) + def append(self, object): self._check_user_storage_quota([object]) return self.locations.append(object) diff --git a/glance/store/__init__.py b/glance/store/__init__.py index e1fba4e015..6d92bc1007 100644 --- a/glance/store/__init__.py +++ b/glance/store/__init__.py @@ -14,6 +14,7 @@ # under the License. import collections +import copy import sys from oslo.config import cfg @@ -622,6 +623,15 @@ class StoreLocations(collections.MutableSequence): def __iter__(self): return iter(self.value) + def __copy__(self): + return type(self)(self.image_proxy, self.value) + + def __deepcopy__(self, memo): + # NOTE(zhiyan): Only copy location entries, others can be reused. + value = copy.deepcopy(self.value, memo) + self.image_proxy.image.locations = value + return type(self)(self.image_proxy, value) + def _locations_proxy(target, attr): """ diff --git a/glance/tests/functional/__init__.py b/glance/tests/functional/__init__.py index c527961c73..d43e3d9ac5 100644 --- a/glance/tests/functional/__init__.py +++ b/glance/tests/functional/__init__.py @@ -322,6 +322,9 @@ class ApiServer(Server): self.user_storage_quota = 0 self.lock_path = self.test_dir + self.location_strategy = 'location_order' + self.store_type_location_strategy_preference = "" + self.conf_base = """[DEFAULT] verbose = %(verbose)s debug = %(debug)s @@ -379,8 +382,11 @@ image_member_quota=%(image_member_quota)s image_property_quota=%(image_property_quota)s image_tag_quota=%(image_tag_quota)s image_location_quota=%(image_location_quota)s +location_strategy=%(location_strategy)s [paste_deploy] flavor = %(deployment_flavor)s +[store_type_location_strategy] +store_type_preference = %(store_type_location_strategy_preference)s """ self.paste_conf_base = """[pipeline:glance-api] pipeline = versionnegotiation gzip unauthenticated-context rootapp diff --git a/glance/tests/functional/v2/test_images.py b/glance/tests/functional/v2/test_images.py index 4c46c64d0b..1c4effaf69 100644 --- a/glance/tests/functional/v2/test_images.py +++ b/glance/tests/functional/v2/test_images.py @@ -14,12 +14,15 @@ # under the License. import os +import signal +import tempfile import uuid import requests from glance.openstack.common import jsonutils from glance.tests import functional +from glance.tests.functional.store import test_http TENANT1 = str(uuid.uuid4()) @@ -1976,6 +1979,160 @@ class TestImageDirectURLVisibility(functional.FunctionalTest): self.stop_servers() +class TestImageLocationSelectionStrategy(functional.FunctionalTest): + + def setUp(self): + super(TestImageLocationSelectionStrategy, self).setUp() + self.cleanup() + self.api_server.deployment_flavor = 'noauth' + self.foo_image_file = tempfile.NamedTemporaryFile() + self.foo_image_file.write("foo image file") + self.foo_image_file.flush() + self.addCleanup(self.foo_image_file.close) + ret = test_http.http_server("foo_image_id", "foo_image") + self.http_server_pid, self.http_port = ret + + def tearDown(self): + if self.http_server_pid is not None: + os.kill(self.http_server_pid, signal.SIGKILL) + + super(TestImageLocationSelectionStrategy, self).tearDown() + + def _url(self, path): + return 'http://127.0.0.1:%d%s' % (self.api_port, path) + + def _headers(self, custom_headers=None): + base_headers = { + 'X-Identity-Status': 'Confirmed', + 'X-Auth-Token': '932c5c84-02ac-4fe5-a9ba-620af0e2bb96', + 'X-User-Id': 'f9a41d13-0c13-47e9-bee2-ce4e8bfe958e', + 'X-Tenant-Id': TENANT1, + 'X-Roles': 'member', + } + base_headers.update(custom_headers or {}) + return base_headers + + def test_image_locations_with_order_strategy(self): + self.api_server.show_image_direct_url = True + self.api_server.show_multiple_locations = True + self.image_location_quota = 10 + self.api_server.location_strategy = 'location_order' + preference = "http, swift, filesystem" + self.api_server.store_type_location_strategy_preference = preference + self.start_servers(**self.__dict__.copy()) + + # Create an image + path = self._url('/v2/images') + headers = self._headers({'content-type': 'application/json'}) + data = jsonutils.dumps({'name': 'image-1', 'type': 'kernel', + 'foo': 'bar', 'disk_format': 'aki', + 'container_format': 'aki'}) + response = requests.post(path, headers=headers, data=data) + self.assertEqual(201, response.status_code) + + # Get the image id + image = jsonutils.loads(response.text) + image_id = image['id'] + + # Image locations should not be visible before location is set + path = self._url('/v2/images/%s' % image_id) + headers = self._headers({'Content-Type': 'application/json'}) + response = requests.get(path, headers=headers) + self.assertEqual(200, response.status_code) + image = jsonutils.loads(response.text) + self.assertTrue('locations' in image) + self.assertTrue(image["locations"] == []) + + # Update image locations via PATCH + path = self._url('/v2/images/%s' % image_id) + media_type = 'application/openstack-images-v2.1-json-patch' + headers = self._headers({'content-type': media_type}) + values = [{'url': 'file://%s' % self.foo_image_file.name, + 'metadata': {'idx': '1'}}, + {'url': 'http://127.0.0.1:%s/foo_image' % self.http_port, + 'metadata': {'idx': '0'}}] + doc = [{'op': 'replace', + 'path': '/locations', + 'value': values}] + data = jsonutils.dumps(doc) + response = requests.patch(path, headers=headers, data=data) + self.assertEqual(200, response.status_code) + + # Image locations should be visible + path = self._url('/v2/images/%s' % image_id) + headers = self._headers({'Content-Type': 'application/json'}) + response = requests.get(path, headers=headers) + self.assertEqual(200, response.status_code) + image = jsonutils.loads(response.text) + self.assertTrue('locations' in image) + self.assertEqual(image['locations'], values) + self.assertTrue('direct_url' in image) + self.assertEqual(image['direct_url'], values[0]['url']) + + self.stop_servers() + + def test_image_locatons_with_store_type_strategy(self): + self.api_server.show_image_direct_url = True + self.api_server.show_multiple_locations = True + self.image_location_quota = 10 + self.api_server.location_strategy = 'store_type' + preference = "http, swift, filesystem" + self.api_server.store_type_location_strategy_preference = preference + self.start_servers(**self.__dict__.copy()) + + # Create an image + path = self._url('/v2/images') + headers = self._headers({'content-type': 'application/json'}) + data = jsonutils.dumps({'name': 'image-1', 'type': 'kernel', + 'foo': 'bar', 'disk_format': 'aki', + 'container_format': 'aki'}) + response = requests.post(path, headers=headers, data=data) + self.assertEqual(201, response.status_code) + + # Get the image id + image = jsonutils.loads(response.text) + image_id = image['id'] + + # Image locations should not be visible before location is set + path = self._url('/v2/images/%s' % image_id) + headers = self._headers({'Content-Type': 'application/json'}) + response = requests.get(path, headers=headers) + self.assertEqual(200, response.status_code) + image = jsonutils.loads(response.text) + self.assertTrue('locations' in image) + self.assertTrue(image["locations"] == []) + + # Update image locations via PATCH + path = self._url('/v2/images/%s' % image_id) + media_type = 'application/openstack-images-v2.1-json-patch' + headers = self._headers({'content-type': media_type}) + values = [{'url': 'file://%s' % self.foo_image_file.name, + 'metadata': {'idx': '1'}}, + {'url': 'http://127.0.0.1:%s/foo_image' % self.http_port, + 'metadata': {'idx': '0'}}] + doc = [{'op': 'replace', + 'path': '/locations', + 'value': values}] + data = jsonutils.dumps(doc) + response = requests.patch(path, headers=headers, data=data) + self.assertEqual(200, response.status_code) + + values.sort(key=lambda loc: int(loc['metadata']['idx'])) + + # Image locations should be visible + path = self._url('/v2/images/%s' % image_id) + headers = self._headers({'Content-Type': 'application/json'}) + response = requests.get(path, headers=headers) + self.assertEqual(200, response.status_code) + image = jsonutils.loads(response.text) + self.assertTrue('locations' in image) + self.assertEqual(image['locations'], values) + self.assertTrue('direct_url' in image) + self.assertEqual(image['direct_url'], values[0]['url']) + + self.stop_servers() + + class TestImageMembers(functional.FunctionalTest): def setUp(self):