diff --git a/doc/source/configuring.rst b/doc/source/configuring.rst index c571a96c59..63c2df6901 100644 --- a/doc/source/configuring.rst +++ b/doc/source/configuring.rst @@ -314,7 +314,7 @@ Optional. Default: ``file`` Can only be specified in configuration files. Sets the storage backend to use by default when storing images in Glance. -Available options for this option are (``file``, ``swift``, ``s3``, ``rbd``, or ``sheepdog``). +Available options for this option are (``file``, ``swift``, ``s3``, ``rbd``, or ``sheepdog``, or ``cinder``). Configuring Glance Image Size Limit ----------------------------------- @@ -702,6 +702,66 @@ Can only be specified in configuration files. Images will be chunked into objects of this size (in megabytes). For best performance, this should be a power of two. +Configuring the Cinder Storage Backend +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Note**: Currently Cinder store is a partial implementation. +After Cinder expose 'brick' library, and 'Readonly-volume-attaching', +'volume-multiple-attaching' enhancement ready, the store will support +'Upload' and 'Download' interface finally. + +* ``cinder_catalog_info=::`` + +Optional. Default: ``volume:cinder:publicURL`` + +Can only be specified in configuration files. + +`This option is specific to the Cinder storage backend.` + +Sets the info to match when looking for cinder in the service catalog. +Format is : separated values of the form: :: + +* ``cinder_endpoint_template=http://ADDR:PORT/VERSION/%(project_id)s`` + +Optional. Default: ``None`` + +Can only be specified in configuration files. + +Override service catalog lookup with template for cinder endpoint. +e.g. http://localhost:8776/v1/%(project_id)s + +* ``os_region_name=REGION_NAME`` + +Optional. Default: ``None`` + +Can only be specified in configuration files. + +Region name of this node. + +* ``cinder_ca_certificates_file=CA_FILE_PATH`` + +Optional. Default: ``None`` + +Can only be specified in configuration files. + +Location of ca certicates file to use for cinder client requests. + +* ``cinder_http_retries=TIMES`` + +Optional. Default: ``3`` + +Can only be specified in configuration files. + +Number of cinderclient retries on failed http calls. + +* ``cinder_api_insecure=ON_OFF`` + +Optional. Default: ``False`` + +Can only be specified in configuration files. + +Allow to perform insecure SSL requests to cinder. + Configuring the Image Cache --------------------------- diff --git a/etc/glance-api.conf b/etc/glance-api.conf index 98f01a2ea7..359ebf5c2c 100644 --- a/etc/glance-api.conf +++ b/etc/glance-api.conf @@ -19,6 +19,7 @@ default_store = file # glance.store.s3.Store, # glance.store.swift.Store, # glance.store.sheepdog.Store, +# glance.store.cinder.Store, # Maximum image size (in bytes) that may be uploaded through the @@ -335,6 +336,30 @@ sheepdog_store_port = 7000 # For best performance, this should be a power of two sheepdog_store_chunk_size = 64 +# ============ Cinder Store Options =============================== + +# Info to match when looking for cinder in the service catalog +# Format is : separated values of the form: +# :: (string value) +#cinder_catalog_info = volume:cinder:publicURL + +# Override service catalog lookup with template for cinder endpoint +# e.g. http://localhost:8776/v1/%(project_id)s (string value) +#cinder_endpoint_template = + +# Region name of this node (string value) +#os_region_name = + +# Location of ca certicates file to use for cinder client requests +# (string value) +#cinder_ca_certificates_file = + +# Number of cinderclient retries on failed http calls (integer value) +#cinder_http_retries = 3 + +# Allow to perform insecure SSL requests to cinder (boolean value) +#cinder_api_insecure = False + # ============ Delayed Delete Options ============================= # Turn on/off delayed delete diff --git a/etc/glance-cache.conf b/etc/glance-cache.conf index 8eec4ca3c4..902bff047a 100644 --- a/etc/glance-cache.conf +++ b/etc/glance-cache.conf @@ -50,6 +50,7 @@ registry_port = 9191 # glance.store.s3.Store, # glance.store.swift.Store, # glance.store.sheepdog.Store, +# glance.store.cinder.Store, # ============ Filesystem Store Options ======================== @@ -135,6 +136,30 @@ s3_store_create_bucket_on_put = False # will be used. If required, an alternative directory can be specified here. # s3_store_object_buffer_dir = /path/to/dir +# ============ Cinder Store Options =========================== + +# Info to match when looking for cinder in the service catalog +# Format is : separated values of the form: +# :: (string value) +#cinder_catalog_info = volume:cinder:publicURL + +# Override service catalog lookup with template for cinder endpoint +# e.g. http://localhost:8776/v1/%(project_id)s (string value) +#cinder_endpoint_template = + +# Region name of this node (string value) +#os_region_name = + +# Location of ca certicates file to use for cinder client requests +# (string value) +#cinder_ca_certificates_file = + +# Number of cinderclient retries on failed http calls (integer value) +#cinder_http_retries = 3 + +# Allow to perform insecure SSL requests to cinder (boolean value) +#cinder_api_insecure = False + # ================= Security Options ========================== # AES key for encrypting store 'location' metadata, including diff --git a/glance/api/v1/images.py b/glance/api/v1/images.py index 946513fb08..5244d1b7c7 100644 --- a/glance/api/v1/images.py +++ b/glance/api/v1/images.py @@ -278,7 +278,7 @@ class Controller(controller.BaseController): If the above constraint is violated, we reject with 400 "Bad Request". """ if source: - for scheme in ['s3', 'swift', 'http', 'rbd', 'sheepdog']: + for scheme in ['s3', 'swift', 'http', 'rbd', 'sheepdog', 'cinder']: if source.lower().startswith(scheme): return source msg = _("External sourcing not supported for store %s") % source diff --git a/glance/common/exception.py b/glance/common/exception.py index ccfcec03d4..62570d1315 100644 --- a/glance/common/exception.py +++ b/glance/common/exception.py @@ -219,6 +219,14 @@ class StoreDeleteNotSupported(GlanceException): message = _("Deleting images from this store is not supported.") +class StoreGetNotSupported(GlanceException): + message = _("Getting images from this store is not supported.") + + +class StoreAddNotSupported(GlanceException): + message = _("Adding images to this store is not supported.") + + class StoreAddDisabled(GlanceException): message = _("Configuration for store failed. Adding images to this " "store is disabled.") diff --git a/glance/store/__init__.py b/glance/store/__init__.py index a76c38afa9..c971639941 100644 --- a/glance/store/__init__.py +++ b/glance/store/__init__.py @@ -42,6 +42,7 @@ store_opts = [ 'glance.store.s3.Store', 'glance.store.swift.Store', 'glance.store.sheepdog.Store', + 'glance.store.cinder.Store', ], help=_('List of which store classes and store class locations ' 'are currently known to glance at startup.')), @@ -226,7 +227,10 @@ def get_from_backend(context, uri, **kwargs): loc = location.get_location_from_uri(uri) store = get_store_from_uri(context, uri, loc) - return store.get(loc) + try: + return store.get(loc) + except NotImplementedError: + raise exception.StoreGetNotSupported def get_size_from_backend(context, uri): @@ -355,7 +359,10 @@ def store_add_to_backend(image_id, data, size, store): def add_to_backend(context, scheme, image_id, data, size): store = get_store_from_scheme(context, scheme) - return store_add_to_backend(image_id, data, size, store) + try: + return store_add_to_backend(image_id, data, size, store) + except NotImplementedError: + raise exception.StoreAddNotSupported def set_acls(context, location_uri, public=False, read_tenants=[], diff --git a/glance/store/cinder.py b/glance/store/cinder.py new file mode 100644 index 0000000000..34a0c83aa7 --- /dev/null +++ b/glance/store/cinder.py @@ -0,0 +1,184 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Storage backend for Cinder""" + +from cinderclient import exceptions as cinder_exception +from cinderclient import service_catalog +from cinderclient.v2 import client as cinderclient + +from oslo.config import cfg + +from glance.common import exception +import glance.openstack.common.log as logging +import glance.openstack.common.uuidutils as uuidutils +import glance.store +import glance.store.base +import glance.store.location + +LOG = logging.getLogger(__name__) + +cinder_opts = [ + cfg.StrOpt('cinder_catalog_info', + default='volume:cinder:publicURL', + help='Info to match when looking for cinder in the service ' + 'catalog. Format is : separated values of the form: ' + '::'), + cfg.StrOpt('cinder_endpoint_template', + default=None, + help='Override service catalog lookup with template for cinder ' + 'endpoint e.g. http://localhost:8776/v1/%(project_id)s'), + cfg.StrOpt('os_region_name', + default=None, + help='Region name of this node'), + cfg.StrOpt('cinder_ca_certificates_file', + default=None, + help='Location of ca certicates file to use for cinder client ' + 'requests.'), + cfg.IntOpt('cinder_http_retries', + default=3, + help='Number of cinderclient retries on failed http calls'), + cfg.BoolOpt('cinder_api_insecure', + default=False, + help='Allow to perform insecure SSL requests to cinder'), +] + +CONF = cfg.CONF +CONF.register_opts(cinder_opts) + + +def get_cinderclient(context): + if CONF.cinder_endpoint_template: + url = CONF.cinder_endpoint_template % context.to_dict() + else: + info = CONF.cinder_catalog_info + service_type, service_name, endpoint_type = info.split(':') + + # extract the region if set in configuration + if CONF.os_region_name: + attr = 'region' + filter_value = CONF.os_region_name + else: + attr = None + filter_value = None + + # FIXME: the cinderclient ServiceCatalog object is mis-named. + # It actually contains the entire access blob. + # Only needed parts of the service catalog are passed in, see + # nova/context.py. + compat_catalog = { + 'access': {'serviceCatalog': context.service_catalog or []}} + sc = service_catalog.ServiceCatalog(compat_catalog) + + url = sc.url_for(attr=attr, + filter_value=filter_value, + service_type=service_type, + service_name=service_name, + endpoint_type=endpoint_type) + + LOG.debug(_('Cinderclient connection created using URL: %s') % url) + + c = cinderclient.Client(context.user, + context.auth_tok, + project_id=context.tenant, + auth_url=url, + insecure=CONF.cinder_api_insecure, + retries=CONF.cinder_http_retries, + cacert=CONF.cinder_ca_certificates_file) + + # noauth extracts user_id:project_id from auth_token + c.client.auth_token = context.auth_tok or '%s:%s' % (context.user, + context.tenant) + c.client.management_url = url + return c + + +class StoreLocation(glance.store.location.StoreLocation): + + """Class describing a Cinder URI""" + + def process_specs(self): + self.scheme = self.specs.get('scheme', 'cinder') + self.volume_id = self.specs.get('volume_id') + + def get_uri(self): + return "cinder://%s" % self.volume_id + + def parse_uri(self, uri): + if not uri.startswith('cinder://'): + reason = _("URI must start with cinder://") + LOG.error(reason) + raise exception.BadStoreUri(uri, reason) + + self.scheme = 'cinder' + self.volume_id = uri[9:] + + if not uuidutils.is_uuid_like(self.volume_id): + reason = _("URI contains invalid volume ID: %s") % self.volume_id + LOG.error(reason) + raise exception.BadStoreUri(uri, reason) + + +class Store(glance.store.base.Store): + + """Cinder backend store adapter.""" + + EXAMPLE_URL = "cinder://volume-id" + + def get_schemes(self): + return ('cinder',) + + def configure_add(self): + """ + Configure the Store to use the stored configuration options + Any store that needs special configuration should implement + this method. If the store was not able to successfully configure + itself, it should raise `exception.BadStoreConfiguration` + """ + + if self.context is None: + reason = _("Cinder storage requires a context.") + raise exception.BadStoreConfiguration(store_name="cinder", + reason=reason) + if self.context.service_catalog is None: + reason = _("Cinder storage requires a service catalog.") + raise exception.BadStoreConfiguration(store_name="cinder", + reason=reason) + + def get_size(self, location): + """ + Takes a `glance.store.location.Location` object that indicates + where to find the image file and returns the image size + + :param location `glance.store.location.Location` object, supplied + from glance.store.location.get_location_from_uri() + :raises `glance.exception.NotFound` if image does not exist + :rtype int + """ + + loc = location.store_location + + try: + volume = get_cinderclient(self.context).volumes.get(loc.volume_id) + # GB unit convert to byte + return volume.size * 1024 * 1024 * 1024 + except cinder_exception.NotFound as e: + reason = _("Failed to get image size due to " + "volume can not be found: %s") % self.volume_id + LOG.error(reason) + raise exception.NotFound(reason) + except Exception as e: + LOG.exception(_("Failed to get image size due to " + "internal error: %s") % e) + return 0 diff --git a/glance/store/location.py b/glance/store/location.py index 7c7d7e765d..f7cb53657e 100644 --- a/glance/store/location.py +++ b/glance/store/location.py @@ -66,6 +66,7 @@ def get_location_from_uri(uri): s3://accesskey:secretkey@s3.amazonaws.com/bucket/key-id s3+https://accesskey:secretkey@s3.amazonaws.com/bucket/key-id file:///var/lib/glance/images/1 + cinder://volume-id """ pieces = urlparse.urlparse(uri) if pieces.scheme not in SCHEME_TO_CLS_MAP.keys(): diff --git a/glance/tests/functional/store/__init__.py b/glance/tests/functional/store/__init__.py index 24ccac1775..64529777fb 100644 --- a/glance/tests/functional/store/__init__.py +++ b/glance/tests/functional/store/__init__.py @@ -65,7 +65,7 @@ class BaseTestCase(object): :param image_data: string representing image data fixture :return URI referencing newly-created backend object """ - raise NotImplementedError('stash_image must be implemented') + raise NotImplementedError('stash_image is not implemented') def test_create_store(self): self.config(known_stores=[self.store_cls_path]) @@ -109,7 +109,12 @@ class BaseTestCase(object): def test_get_remote_image(self): """Get an image that was created externally to Glance""" image_id = uuidutils.generate_uuid() - image_uri = self.stash_image(image_id, 'XXX') + try: + image_uri = self.stash_image(image_id, 'XXX') + except NotImplementedError: + msg = 'Configured store can not stash images' + self.skipTest(msg) + store = self.get_store() location = glance.store.location.Location( self.store_name, diff --git a/glance/tests/functional/store/test_cinder.py b/glance/tests/functional/store/test_cinder.py new file mode 100644 index 0000000000..9402acd3af --- /dev/null +++ b/glance/tests/functional/store/test_cinder.py @@ -0,0 +1,95 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 OpenStack, LLC +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Functional tests for the Cinder store interface +""" + +import os + +import oslo.config.cfg +import testtools + +import glance.store.cinder as cinder +import glance.tests.functional.store as store_tests +import glance.tests.functional.store.test_swift as store_tests_swift +import glance.tests.utils + + +def parse_config(config): + out = {} + options = [ + 'test_cinder_store_auth_address', + 'test_cinder_store_auth_version', + 'test_cinder_store_tenant', + 'test_cinder_store_user', + 'test_cinder_store_key', + ] + + for option in options: + out[option] = config.defaults()[option] + + return out + + +class TestCinderStore(store_tests.BaseTestCase, testtools.TestCase): + + store_cls_path = 'glance.store.cinder.Store' + store_cls = glance.store.cinder.Store + store_name = 'cinder' + + def setUp(self): + config_path = os.environ.get('GLANCE_TEST_CINDER_CONF') + if not config_path: + msg = "GLANCE_TEST_CINDER_CONF environ not set." + self.skipTest(msg) + + oslo.config.cfg.CONF(args=[], default_config_files=[config_path]) + raw_config = store_tests_swift.read_config(config_path) + + try: + self.cinder_config = parse_config(raw_config) + ret = store_tests_swift.keystone_authenticate( + self.cinder_config['test_cinder_store_auth_address'], + self.cinder_config['test_cinder_store_auth_version'], + self.cinder_config['test_cinder_store_tenant'], + self.cinder_config['test_cinder_store_user'], + self.cinder_config['test_cinder_store_key']) + (tenant_id, auth_token, service_catalog) = ret + self.context = glance.context.RequestContext( + tenant=tenant_id, + service_catalog=service_catalog, + auth_tok=auth_token) + self.cinder_client = cinder.get_cinderclient(self.context) + except Exception as e: + msg = "Cinder backend isn't set up: %s" % e + self.skipTest(msg) + + super(TestCinderStore, self).setUp() + + def get_store(self, **kwargs): + store = cinder.Store(context=kwargs.get('context') or self.context) + store.configure() + store.configure_add() + return store + + def stash_image(self, image_id, image_data): + #(zhiyan): Currently cinder store is a partial implementation, + # after Cinder expose 'brick' library, 'host-volume-attaching' and + # 'multiple-attaching' enhancement ready, the store will support + # ADD/GET/DELETE interface. + raise NotImplementedError('stash_image can not be implemented so far') diff --git a/glance/tests/unit/test_cinder_store.py b/glance/tests/unit/test_cinder_store.py new file mode 100644 index 0000000000..9009fe0e2b --- /dev/null +++ b/glance/tests/unit/test_cinder_store.py @@ -0,0 +1,84 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 OpenStack, LLC +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import stubout + +from cinderclient.v2 import client as cinderclient + +from glance.common import exception +import glance.store.cinder as cinder +from glance.store.location import get_location_from_uri +from glance.tests.unit import base + + +class FakeObject(object): + def __init__(self, **kwargs): + for name, value in kwargs.iteritems(): + setattr(self, name, value) + + +class TestCinderStore(base.StoreClearingUnitTest): + + def setUp(self): + self.config(default_store='cinder', + known_stores=['glance.store.cinder.Store']) + super(TestCinderStore, self).setUp() + self.stubs = stubout.StubOutForTesting() + + def test_cinder_configure_add(self): + store = cinder.Store() + self.assertRaises(exception.BadStoreConfiguration, + store.configure_add) + store = cinder.Store(context=None) + self.assertRaises(exception.BadStoreConfiguration, + store.configure_add) + store = cinder.Store(context=FakeObject(service_catalog=None)) + self.assertRaises(exception.BadStoreConfiguration, + store.configure_add) + store = cinder.Store(context=FakeObject(service_catalog= + 'fake_service_catalog')) + store.configure_add() + + def test_cinder_get_size(self): + fake_client = FakeObject(auth_token=None, management_url=None) + fake_volumes = {'12345678-9012-3455-6789-012345678901': + FakeObject(size=5)} + + class FakeCinderClient(FakeObject): + def __init__(self, *args, **kwargs): + super(FakeCinderClient, self).__init__(client=fake_client, + volumes=fake_volumes) + + self.stubs.Set(cinderclient, 'Client', FakeCinderClient) + + fake_sc = [{u'endpoints': [{u'publicURL': u'foo_public_url'}], + u'endpoints_links': [], + u'name': u'cinder', + u'type': u'volume'}] + fake_context = FakeObject(service_catalog=fake_sc, + user='fake_uer', + auth_tok='fake_token', + tenant='fake_tenant') + + uri = 'cinder://%s' % fake_volumes.keys()[0] + loc = get_location_from_uri(uri) + store = cinder.Store(context=fake_context) + image_size = store.get_size(loc) + self.assertEqual(image_size, + fake_volumes.values()[0].size * 1024 * 1024 * 1024) + self.assertEqual(fake_client.auth_token, 'fake_token') + self.assertEqual(fake_client.management_url, 'foo_public_url') diff --git a/glance/tests/unit/test_store_location.py b/glance/tests/unit/test_store_location.py index 79f6ee290f..7eec171139 100644 --- a/glance/tests/unit/test_store_location.py +++ b/glance/tests/unit/test_store_location.py @@ -1,6 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2011 OpenStack, LLC +# Copyright 2011-2013 OpenStack, LLC # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -53,6 +53,7 @@ class TestStoreLocation(base.StoreClearingUnitTest): 'rbd://fsid/pool/image/snap', 'rbd://%2F/%2F/%2F/%2F', 'sheepdog://imagename', + 'cinder://12345678-9012-3455-6789-012345678901', ] for uri in good_store_uris: @@ -377,6 +378,23 @@ class TestStoreLocation(base.StoreClearingUnitTest): bad_uri = 'http://image' self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri) + def test_cinder_store_good_location(self): + """ + Test the specific StoreLocation for the Cinder store + """ + good_uri = 'cinder://12345678-9012-3455-6789-012345678901' + loc = glance.store.cinder.StoreLocation({}) + loc.parse_uri(good_uri) + self.assertEqual('12345678-9012-3455-6789-012345678901', loc.volume_id) + + def test_cinder_store_bad_location(self): + """ + Test the specific StoreLocation for the Cinder store + """ + bad_uri = 'cinder://volume-id-is-a-uuid' + loc = glance.store.cinder.StoreLocation({}) + self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri) + def test_get_store_from_scheme(self): """ Test that the backend returned by glance.store.get_backend_class @@ -394,7 +412,8 @@ class TestStoreLocation(base.StoreClearingUnitTest): 'http': glance.store.http.Store, 'https': glance.store.http.Store, 'rbd': glance.store.rbd.Store, - 'sheepdog': glance.store.sheepdog.Store} + 'sheepdog': glance.store.sheepdog.Store, + 'cinder': glance.store.cinder.Store} ctx = context.RequestContext() for scheme, store in good_results.items(): diff --git a/tools/pip-requires b/tools/pip-requires index 3863061144..f8ae7931f0 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -35,6 +35,7 @@ Paste passlib jsonschema +python-cinderclient>=1.0.4 python-keystoneclient>=0.2.0 pyOpenSSL