diff --git a/glance/api/v1/controller.py b/glance/api/v1/controller.py index 5b26789073..63366ad8a4 100644 --- a/glance/api/v1/controller.py +++ b/glance/api/v1/controller.py @@ -15,10 +15,12 @@ import webob.exc +import glance_store as store + from glance.common import exception import glance.openstack.common.log as logging import glance.registry.client.v1.api as registry -import glance.store as store + LOG = logging.getLogger(__name__) @@ -74,10 +76,11 @@ class BaseController(object): write_tenants.append(member['member_id']) else: read_tenants.append(member['member_id']) - store.set_acls(req.context, location_uri, public=public, + store.set_acls(location_uri, public=public, read_tenants=read_tenants, - write_tenants=write_tenants) - except exception.UnknownScheme: + write_tenants=write_tenants, + context=req.context) + except store.UnknownScheme: msg = _("Store for image_id not found: %s") % image_id raise webob.exc.HTTPBadRequest(explanation=msg, request=req, diff --git a/glance/api/v1/images.py b/glance/api/v1/images.py index c448cc5298..7ec3bfc594 100644 --- a/glance/api/v1/images.py +++ b/glance/api/v1/images.py @@ -20,6 +20,8 @@ import copy import eventlet +import glance_store as store +import glance_store.location from oslo.config import cfg import six.moves.urllib.parse as urlparse from webob.exc import HTTPBadRequest @@ -47,12 +49,6 @@ from glance.openstack.common import gettextutils import glance.openstack.common.log as logging from glance.openstack.common import strutils import glance.registry.client.v1.api as registry -from glance.store import get_from_backend -from glance.store import get_known_schemes -from glance.store import get_size_from_backend -from glance.store import get_store_from_location -from glance.store import get_store_from_scheme -from glance.store import validate_location LOG = logging.getLogger(__name__) _LI = gettextutils._LI @@ -425,7 +421,7 @@ class Controller(controller.BaseController): """ if source: pieces = urlparse.urlparse(source) - schemes = [scheme for scheme in get_known_schemes() + schemes = [scheme for scheme in store.get_known_schemes() if scheme != 'file'] for scheme in schemes: if pieces.scheme == scheme: @@ -451,8 +447,14 @@ class Controller(controller.BaseController): @staticmethod def _get_from_store(context, where, dest=None): try: - image_data, image_size = get_from_backend( - context, where, dest=dest) + loc = glance_store.location.get_location_from_uri(where) + src_store = store.get_store_from_uri(where) + + if dest is not None: + src_store.READ_CHUNKSIZE = dest.WRITE_CHUNKSIZE + + image_data, image_size = src_store.get(loc, context=context) + except exception.NotFound as e: raise HTTPNotFound(explanation=e.msg) image_size = int(image_size) if image_size else None @@ -492,7 +494,6 @@ class Controller(controller.BaseController): image_meta['location']) image_iterator = utils.cooperative_iter(image_iterator) image_meta['size'] = size or image_meta['size'] - image_meta = redact_loc(image_meta) return { 'image_iterator': image_iterator, @@ -513,9 +514,9 @@ class Controller(controller.BaseController): :raises HTTPBadRequest if image metadata is not valid """ location = self._external_source(image_meta, req) - store = image_meta.get('store') - if store and store not in get_known_schemes(): - msg = "Required store %s is invalid" % store + scheme = image_meta.get('store') + if scheme and scheme not in store.get_known_schemes(): + msg = "Required store %s is invalid" % scheme LOG.debug(msg) raise HTTPBadRequest(explanation=msg, content_type='text/plain') @@ -525,8 +526,8 @@ class Controller(controller.BaseController): if location: try: - store = get_store_from_location(location) - except exception.BadStoreUri: + backend = store.get_store_from_location(location) + except store.BadStoreUri: msg = "Invalid location %s" % location LOG.debug(msg) raise HTTPBadRequest(explanation=msg, @@ -534,7 +535,7 @@ class Controller(controller.BaseController): content_type="text/plain") # check the store exists before we hit the registry, but we # don't actually care what it is at this point - self.get_store_or_400(req, store) + self.get_store_or_400(req, backend) # retrieve the image size from remote store (if not provided) image_meta['size'] = self._get_size(req.context, image_meta, @@ -585,7 +586,9 @@ class Controller(controller.BaseController): :retval The location where the image was stored """ - scheme = req.headers.get('x-image-meta-store', CONF.default_store) + scheme = req.headers.get('x-image-meta-store', + CONF.glance_store.default_store) + store = self.get_store_or_400(req, scheme) copy_from = self._copy_from(req) @@ -691,8 +694,8 @@ class Controller(controller.BaseController): # retrieve the image size from remote store (if not provided) try: return (image_meta.get('size', 0) or - get_size_from_backend(context, location)) - except (exception.NotFound, exception.BadStoreUri) as e: + store.get_size_from_backend(location, context=context)) + except (exception.NotFound, store.BadStoreUri) as e: LOG.debug(e) raise HTTPBadRequest(explanation=e.msg, content_type="text/plain") @@ -718,16 +721,17 @@ class Controller(controller.BaseController): else: if location: try: - validate_location(req.context, location) - except exception.BadStoreUri as bse: + store.validate_location(location, context=req.context) + except store.BadStoreUri as bse: raise HTTPBadRequest(explanation=bse.msg, request=req) self._validate_image_for_activation(req, image_id, image_meta) image_size_meta = image_meta.get('size') if image_size_meta: - image_size_store = get_size_from_backend(req.context, - location) + image_size_store = store.get_size_from_backend( + location, + context=req.context) # NOTE(zhiyan): A returned size of zero usually means # the driver encountered an error. In this case the # size provided by the client will be used as-is. @@ -902,7 +906,7 @@ class Controller(controller.BaseController): try: self.update_store_acls(req, id, orig_or_updated_loc, public=is_public) - except exception.BadStoreUri: + except store.BadStoreUri: msg = "Invalid location %s" % location LOG.debug(msg) raise HTTPBadRequest(explanation=msg, @@ -1049,6 +1053,7 @@ class Controller(controller.BaseController): with excutils.save_and_reraise_exception(): registry.update_image_metadata(req.context, id, {'status': ori_status}) + registry.delete_image_metadata(req.context, id) except exception.NotFound as e: msg = (_("Failed to find image to delete: %s") % @@ -1089,7 +1094,7 @@ class Controller(controller.BaseController): :raises HTTPNotFound if store does not exist """ try: - return get_store_from_scheme(request.context, scheme) + return store.get_store_from_scheme(scheme) except exception.UnknownScheme: msg = "Store for scheme %s not found" % scheme LOG.debug(msg) diff --git a/glance/api/v1/upload_utils.py b/glance/api/v1/upload_utils.py index 439b50abbc..2e22dae3d0 100644 --- a/glance/api/v1/upload_utils.py +++ b/glance/api/v1/upload_utils.py @@ -16,6 +16,8 @@ from oslo.config import cfg import webob.exc +import glance_store as store_api + from glance.common import exception from glance.common import store_utils from glance.common import utils @@ -24,7 +26,6 @@ from glance.openstack.common import excutils from glance.openstack.common import gettextutils import glance.openstack.common.log as logging import glance.registry.client.v1.api as registry -import glance.store as store_api CONF = cfg.CONF @@ -192,7 +193,7 @@ def upload_data_to_store(req, image_meta, image_data, store, notifier): request=req, content_type="text/plain") - except exception.StorageFull as e: + except store_api.StorageFull as e: msg = _("Image storage media is full: %s") % utils.exception_to_str(e) LOG.error(msg) safe_kill(req, image_id, 'saving') @@ -201,7 +202,7 @@ def upload_data_to_store(req, image_meta, image_data, store, notifier): request=req, content_type='text/plain') - except exception.StorageWriteDenied as e: + except store_api.StorageWriteDenied as e: msg = (_("Insufficient permissions on image storage media: %s") % utils.exception_to_str(e)) LOG.error(msg) diff --git a/glance/api/v2/image_data.py b/glance/api/v2/image_data.py index a87e37ea3d..cccf141989 100644 --- a/glance/api/v2/image_data.py +++ b/glance/api/v2/image_data.py @@ -15,6 +15,8 @@ import webob.exc +import glance_store + import glance.api.policy from glance.common import exception from glance.common import utils @@ -25,7 +27,7 @@ import glance.notifier from glance.openstack.common import excutils from glance.openstack.common import gettextutils import glance.openstack.common.log as logging -import glance.store + LOG = logging.getLogger(__name__) _LE = gettextutils._LE @@ -37,7 +39,7 @@ class ImageDataController(object): gateway=None): if gateway is None: db_api = db_api or glance.db.get_api() - store_api = store_api or glance.store + store_api = store_api or glance_store policy = policy_enforcer or glance.api.policy.Enforcer() notifier = notifier or glance.notifier.Notifier() gateway = glance.gateway.Gateway(db_api, store_api, @@ -110,7 +112,7 @@ class ImageDataController(object): except exception.NotFound as e: raise webob.exc.HTTPNotFound(explanation=e.msg) - except exception.StorageFull as e: + except glance_store.StorageFull as e: msg = _("Image storage media " "is full: %s") % utils.exception_to_str(e) LOG.error(msg) @@ -134,7 +136,7 @@ class ImageDataController(object): raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg, request=req) - except exception.StorageWriteDenied as e: + except glance_store.StorageWriteDenied as e: msg = _("Insufficient permissions on image " "storage media: %s") % utils.exception_to_str(e) LOG.error(msg) diff --git a/glance/api/v2/image_members.py b/glance/api/v2/image_members.py index 0d83157251..bc1c13dbbf 100644 --- a/glance/api/v2/image_members.py +++ b/glance/api/v2/image_members.py @@ -17,6 +17,8 @@ import copy import six import webob +import glance_store + from glance.api import policy from glance.common import exception from glance.common import utils @@ -27,7 +29,6 @@ import glance.notifier from glance.openstack.common import jsonutils from glance.openstack.common import timeutils import glance.schema -import glance.store class ImageMembersController(object): @@ -36,7 +37,7 @@ class ImageMembersController(object): self.db_api = db_api or glance.db.get_api() self.policy = policy_enforcer or policy.Enforcer() self.notifier = notifier or glance.notifier.Notifier() - self.store_api = store_api or glance.store + self.store_api = store_api or glance_store self.gateway = glance.gateway.Gateway(self.db_api, self.store_api, self.notifier, self.policy) diff --git a/glance/api/v2/image_tags.py b/glance/api/v2/image_tags.py index 9663a1e0eb..277bf66e08 100644 --- a/glance/api/v2/image_tags.py +++ b/glance/api/v2/image_tags.py @@ -15,6 +15,8 @@ import webob.exc +import glance_store + from glance.api import policy from glance.common import exception from glance.common import utils @@ -22,7 +24,6 @@ from glance.common import wsgi import glance.db import glance.gateway import glance.notifier -import glance.store class Controller(object): @@ -31,7 +32,7 @@ class Controller(object): self.db_api = db_api or glance.db.get_api() self.policy = policy_enforcer or policy.Enforcer() self.notifier = notifier or glance.notifier.Notifier() - self.store_api = store_api or glance.store + self.store_api = store_api or glance_store self.gateway = glance.gateway.Gateway(self.db_api, self.store_api, self.notifier, self.policy) diff --git a/glance/api/v2/images.py b/glance/api/v2/images.py index cabed3a3a2..084ec5bdbd 100644 --- a/glance/api/v2/images.py +++ b/glance/api/v2/images.py @@ -15,6 +15,7 @@ import re +import glance_store from oslo.config import cfg import six import six.moves.urllib.parse as urlparse @@ -33,7 +34,6 @@ from glance.openstack.common import jsonutils as json import glance.openstack.common.log as logging from glance.openstack.common import timeutils import glance.schema -import glance.store LOG = logging.getLogger(__name__) _LI = gettextutils._LI @@ -51,7 +51,7 @@ class ImagesController(object): self.db_api = db_api or glance.db.get_api() self.policy = policy_enforcer or policy.Enforcer() self.notifier = notifier or glance.notifier.Notifier() - self.store_api = store_api or glance.store + self.store_api = store_api or glance_store self.gateway = glance.gateway.Gateway(self.db_api, self.store_api, self.notifier, self.policy) diff --git a/glance/api/v2/metadef_namespaces.py b/glance/api/v2/metadef_namespaces.py index 64ead58e5f..6e334faecd 100644 --- a/glance/api/v2/metadef_namespaces.py +++ b/glance/api/v2/metadef_namespaces.py @@ -37,7 +37,6 @@ import glance.notifier from glance.openstack.common import jsonutils as json import glance.openstack.common.log as logging import glance.schema -import glance.store LOG = logging.getLogger(__name__) _LE = i18n._LE diff --git a/glance/api/v2/metadef_properties.py b/glance/api/v2/metadef_properties.py index 33c2d41c4e..5f2760695e 100644 --- a/glance/api/v2/metadef_properties.py +++ b/glance/api/v2/metadef_properties.py @@ -33,7 +33,6 @@ import glance.notifier from glance.openstack.common import jsonutils as json import glance.openstack.common.log as logging import glance.schema -import glance.store LOG = logging.getLogger(__name__) _LE = i18n._LE diff --git a/glance/api/v2/metadef_resource_types.py b/glance/api/v2/metadef_resource_types.py index f6be343895..73487e36cd 100644 --- a/glance/api/v2/metadef_resource_types.py +++ b/glance/api/v2/metadef_resource_types.py @@ -33,7 +33,6 @@ import glance.notifier from glance.openstack.common import jsonutils as json import glance.openstack.common.log as logging import glance.schema -import glance.store LOG = logging.getLogger(__name__) _LE = i18n._LE diff --git a/glance/api/v2/tasks.py b/glance/api/v2/tasks.py index f744dc9f74..86ca23b7e0 100644 --- a/glance/api/v2/tasks.py +++ b/glance/api/v2/tasks.py @@ -17,6 +17,7 @@ import copy import webob.exc +import glance_store from oslo.config import cfg import six import six.moves.urllib.parse as urlparse @@ -33,7 +34,6 @@ import glance.openstack.common.jsonutils as json import glance.openstack.common.log as logging from glance.openstack.common import timeutils import glance.schema -import glance.store LOG = logging.getLogger(__name__) _LI = gettextutils._LI @@ -50,7 +50,7 @@ class TasksController(object): self.db_api = db_api or glance.db.get_api() self.policy = policy_enforcer or policy.Enforcer() self.notifier = notifier or glance.notifier.Notifier() - self.store_api = store_api or glance.store + self.store_api = store_api or glance_store self.gateway = glance.gateway.Gateway(self.db_api, self.store_api, self.notifier, self.policy) diff --git a/glance/cmd/api.py b/glance/cmd/api.py index d508d6e4e7..557dc3b05f 100755 --- a/glance/cmd/api.py +++ b/glance/cmd/api.py @@ -39,6 +39,7 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'glance', '__init__.py')): sys.path.insert(0, possible_topdir) +import glance_store from oslo.config import cfg import osprofiler.notifier import osprofiler.web @@ -48,7 +49,6 @@ from glance.common import exception from glance.common import wsgi from glance import notifier from glance.openstack.common import log -import glance.store CONF = cfg.CONF CONF.import_group("profiler", "glance.common.wsgi") @@ -65,8 +65,9 @@ def main(): wsgi.set_eventlet_hub() log.setup('glance') - glance.store.create_stores() - glance.store.verify_default_store() + glance_store.register_opts(config.CONF) + glance_store.create_stores(config.CONF) + glance_store.verify_default_store() if cfg.CONF.profiler.enabled: _notifier = osprofiler.notifier.create("Messaging", diff --git a/glance/cmd/cache_prefetcher.py b/glance/cmd/cache_prefetcher.py index 0a2c6acbb0..eab2319fc1 100755 --- a/glance/cmd/cache_prefetcher.py +++ b/glance/cmd/cache_prefetcher.py @@ -33,10 +33,11 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'glance', '__init__.py')): sys.path.insert(0, possible_topdir) +import glance_store + from glance.common import config from glance.image_cache import prefetcher from glance.openstack.common import log -import glance.store def main(): @@ -44,8 +45,9 @@ def main(): config.parse_cache_args() log.setup('glance') - glance.store.create_stores() - glance.store.verify_default_store() + glance_store.register_opts(config.CONF) + glance_store.create_stores(config.CONF) + glance_store.verify_default_store() app = prefetcher.Prefetcher() app.run() diff --git a/glance/cmd/scrubber.py b/glance/cmd/scrubber.py index 95694ea9bd..7f97c5d41d 100755 --- a/glance/cmd/scrubber.py +++ b/glance/cmd/scrubber.py @@ -30,12 +30,13 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'glance', '__init__.py')): sys.path.insert(0, possible_topdir) +import glance_store from oslo.config import cfg from glance.common import config from glance.openstack.common import log from glance import scrubber -import glance.store + CONF = cfg.CONF @@ -57,10 +58,11 @@ def main(): config.parse_args() log.setup('glance') - glance.store.create_stores() - glance.store.verify_default_store() + glance_store.register_opts(config.CONF) + glance_store.create_stores(config.CONF) + glance_store.verify_default_store() - app = scrubber.Scrubber(glance.store) + app = scrubber.Scrubber(glance_store) if CONF.daemon: server = scrubber.Daemon(CONF.wakeup_time) diff --git a/glance/common/exception.py b/glance/common/exception.py index 6155dd3d5c..a910c14173 100644 --- a/glance/common/exception.py +++ b/glance/common/exception.py @@ -72,10 +72,6 @@ class NotFound(GlanceException): message = _("An object with the specified identifier was not found.") -class UnknownScheme(GlanceException): - message = _("Unknown scheme '%(scheme)s' found in URI") - - class BadStoreUri(GlanceException): message = _("The Store URI was malformed.") @@ -89,19 +85,11 @@ class Conflict(GlanceException): "operated on.") -class StorageFull(GlanceException): - message = _("There is not enough disk space on the image storage media.") - - class StorageQuotaFull(GlanceException): message = _("The size of the data %(image_size)s will exceed the limit. " "%(remaining)s bytes remaining.") -class StorageWriteDenied(GlanceException): - message = _("Permission to write image storage media denied.") - - class AuthBadRequest(GlanceException): message = _("Connect error/bad request to Auth service at URL %(url)s.") @@ -241,37 +229,11 @@ class BadRegistryConnectionConfiguration(GlanceException): "Reason: %(reason)s") -class BadStoreConfiguration(GlanceException): - message = _("Store %(store_name)s could not be configured correctly. " - "Reason: %(reason)s") - - class BadDriverConfiguration(GlanceException): message = _("Driver %(driver_name)s could not be configured correctly. " "Reason: %(reason)s") -class StoreDeleteNotSupported(GlanceException): - message = _("Deleting images from this store is not supported.") - - -class StoreGetNotSupported(GlanceException): - message = _("Getting images from this store is not supported.") - - -class StoreAddNotSupported(GlanceException): - message = _("Adding images to this store is not supported.") - - -class StoreAddDisabled(GlanceException): - message = _("Configuration for store failed. Adding images to this " - "store is disabled.") - - -class StoreNotConfigured(GlanceException): - message = _("Store is not configured.") - - class MaxRedirectsExceeded(GlanceException): message = _("Maximum redirects (%(redirects)s) was exceeded.") diff --git a/glance/common/location_strategy/__init__.py b/glance/common/location_strategy/__init__.py index 55a7d55279..45f90115ac 100644 --- a/glance/common/location_strategy/__init__.py +++ b/glance/common/location_strategy/__init__.py @@ -66,7 +66,7 @@ def _load_strategies(): _available_strategies = _load_strategies() -# TODO(kadachi): Not used but don't remove this until glance.store +# TODO(kadachi): Not used but don't remove this until glance_store # development/migration stage. def verify_location_strategy(conf=None, strategies=_available_strategies): """Validate user configured 'location_strategy' option value.""" diff --git a/glance/common/store_utils.py b/glance/common/store_utils.py index bef04ceb6b..8f04d39a9d 100644 --- a/glance/common/store_utils.py +++ b/glance/common/store_utils.py @@ -14,15 +14,14 @@ import sys +import glance_store as store_api from oslo.config import cfg -from glance.common import exception from glance.common import utils import glance.db as db_api from glance.openstack.common import gettextutils import glance.openstack.common.log as logging from glance import scrubber -import glance.store as store_api _LE = gettextutils._LE _LW = gettextutils._LW @@ -53,16 +52,16 @@ def safe_delete_from_backend(context, image_id, location): """ try: - ret = store_api.delete_from_backend(context, location['url']) + ret = store_api.delete_from_backend(location['url'], context=context) location['status'] = 'deleted' if 'id' in location: db_api.get_api().image_location_delete(context, image_id, location['id'], 'deleted') return ret - except exception.NotFound: + except store_api.NotFound: msg = _LW('Failed to delete image %s in store from URI') % image_id LOG.warn(msg) - except exception.StoreDeleteNotSupported as e: + except store_api.StoreDeleteNotSupported as e: LOG.warn(utils.exception_to_str(e)) except store_api.UnsupportedBackend: exc_type = sys.exc_info()[0].__name__ diff --git a/glance/db/sqlalchemy/migrate_repo/versions/017_quote_encrypted_swift_credentials.py b/glance/db/sqlalchemy/migrate_repo/versions/017_quote_encrypted_swift_credentials.py index 780b266eee..dc9ba525f2 100644 --- a/glance/db/sqlalchemy/migrate_repo/versions/017_quote_encrypted_swift_credentials.py +++ b/glance/db/sqlalchemy/migrate_repo/versions/017_quote_encrypted_swift_credentials.py @@ -28,6 +28,8 @@ Fixes bug #1081043 """ import types # noqa +#NOTE(flaper87): This is bad but there ain't better way to do it. +from glance_store._drivers import swift # noqa from oslo.config import cfg import six.moves.urllib.parse as urlparse import sqlalchemy @@ -37,7 +39,6 @@ from glance.common import exception from glance.common import utils from glance.openstack.common import gettextutils import glance.openstack.common.log as logging -import glance.store.swift # noqa LOG = logging.getLogger(__name__) _LE = gettextutils._LE diff --git a/glance/domain/__init__.py b/glance/domain/__init__.py index 71df1b86e3..4f4fb9c053 100644 --- a/glance/domain/__init__.py +++ b/glance/domain/__init__.py @@ -38,12 +38,12 @@ _delayed_delete_imported = False def _import_delayed_delete(): - # glance.store (indirectly) imports glance.domain therefore we can't put + # glance_store (indirectly) imports glance.domain therefore we can't put # the CONF.import_opt outside - we have to do it in a convoluted/indirect # way! global _delayed_delete_imported if not _delayed_delete_imported: - CONF.import_opt('delayed_delete', 'glance.store') + CONF.import_opt('delayed_delete', 'glance_store') _delayed_delete_imported = True diff --git a/glance/gateway.py b/glance/gateway.py index 9a19c244c0..1b14061ab8 100644 --- a/glance/gateway.py +++ b/glance/gateway.py @@ -24,14 +24,14 @@ import glance.domain import glance.location import glance.notifier import glance.quota -import glance.store +import glance_store class Gateway(object): def __init__(self, db_api=None, store_api=None, notifier=None, policy_enforcer=None): self.db_api = db_api or glance.db.get_api() - self.store_api = store_api or glance.store + self.store_api = store_api or glance_store self.store_utils = store_utils self.notifier = notifier or glance.notifier.Notifier() self.policy = policy_enforcer or policy.Enforcer() diff --git a/glance/image_cache/prefetcher.py b/glance/image_cache/prefetcher.py index b77866f3f4..ce9f66ef3b 100644 --- a/glance/image_cache/prefetcher.py +++ b/glance/image_cache/prefetcher.py @@ -19,13 +19,14 @@ Prefetches images into the Image Cache import eventlet +import glance_store + from glance.common import exception from glance import context from glance.image_cache import base from glance.openstack.common import gettextutils import glance.openstack.common.log as logging import glance.registry.client.v1.api as registry -import glance.store LOG = logging.getLogger(__name__) _LI = gettextutils._LI @@ -54,8 +55,9 @@ class Prefetcher(base.CacheApp): return False location = image_meta['location'] - image_data, image_size = glance.store.get_from_backend(ctx, location) - LOG.debug("Caching image '%s'" % image_id) + image_data, image_size = glance_store.get_from_backend(location, + context=ctx) + LOG.debug("Caching image '%s'", image_id) cache_tee_iter = self.cache.cache_tee_iter(image_id, image_data, image_meta['checksum']) # Image is tee'd into cache and checksum verified diff --git a/glance/location.py b/glance/location.py index 9fe06053e6..a5e0c76f80 100644 --- a/glance/location.py +++ b/glance/location.py @@ -16,6 +16,7 @@ import collections import copy +import glance_store as store from oslo.config import cfg from glance.common import exception @@ -24,7 +25,7 @@ import glance.domain.proxy from glance.openstack.common import excutils from glance.openstack.common import gettextutils import glance.openstack.common.log as logging -from glance import store + _LE = gettextutils._LE @@ -50,8 +51,9 @@ class ImageRepoProxy(glance.domain.proxy.Repo): member_repo = image.get_member_repo() member_ids = [m.member_id for m in member_repo.list()] for location in image.locations: - self.store_api.set_acls(self.context, location['url'], public, - read_tenants=member_ids) + self.store_api.set_acls(location['url'], public=public, + read_tenants=member_ids, + context=self.context) def add(self, image): result = super(ImageRepoProxy, self).add(image) @@ -73,10 +75,10 @@ def _check_location_uri(context, store_api, uri): """ is_ok = True try: - size = store_api.get_size_from_backend(context, uri) + size = store_api.get_size_from_backend(uri, context=context) # NOTE(zhiyan): Some stores return zero when it catch exception is_ok = size > 0 - except (exception.UnknownScheme, exception.NotFound): + except (store.UnknownScheme, store.NotFound): is_ok = False if not is_ok: reason = _('Invalid location') @@ -92,7 +94,8 @@ def _set_image_size(context, image, locations): if not image.size: for location in locations: size_from_backend = store.get_size_from_backend( - context, location['url']) + location['url'], context=context) + if size_from_backend: # NOTE(flwang): This assumes all locations have the same size image.size = size_from_backend @@ -353,11 +356,12 @@ class ImageProxy(glance.domain.proxy.Image): if size is None: size = 0 # NOTE(markwash): zero -> unknown size location, size, checksum, loc_meta = self.store_api.add_to_backend( - self.context, CONF.default_store, + CONF, self.image.image_id, utils.LimitingReader(utils.CooperativeReader(data), CONF.image_size_cap), - size) + size, + context=self.context) self.image.locations = [{'url': location, 'metadata': loc_meta, 'status': 'active'}] self.image.size = size @@ -366,12 +370,13 @@ class ImageProxy(glance.domain.proxy.Image): def get_data(self): if not self.image.locations: - raise exception.NotFound(_("No image data could be found")) + raise store.NotFound(_("No image data could be found")) err = None for loc in self.image.locations: try: - data, size = self.store_api.get_from_backend(self.context, - loc['url']) + data, size = self.store_api.get_from_backend( + loc['url'], + context=self.context) return data except Exception as e: @@ -398,8 +403,9 @@ class ImageMemberRepoProxy(glance.domain.proxy.Repo): if self.image.locations and not public: member_ids = [m.member_id for m in self.repo.list()] for location in self.image.locations: - self.store_api.set_acls(self.context, location['url'], - public, read_tenants=member_ids) + self.store_api.set_acls(location['url'], public=public, + read_tenants=member_ids, + context=self.context) def add(self, member): super(ImageMemberRepoProxy, self).add(member) diff --git a/glance/notifier.py b/glance/notifier.py index 8e54e48656..9800ffa6b0 100644 --- a/glance/notifier.py +++ b/glance/notifier.py @@ -14,6 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. +import glance_store from oslo.config import cfg from oslo import messaging import webob @@ -195,12 +196,12 @@ class ImageProxy(glance.domain.proxy.Image): self.notifier.info('image.prepare', payload) try: self.image.set_data(data, size) - except exception.StorageFull as e: + except glance_store.StorageFull as e: msg = (_("Image storage media is full: %s") % utils.exception_to_str(e)) self.notifier.error('image.upload', msg) raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg) - except exception.StorageWriteDenied as e: + except glance_store.StorageWriteDenied as e: msg = (_("Insufficient permissions on image storage media: %s") % utils.exception_to_str(e)) self.notifier.error('image.upload', msg) diff --git a/glance/quota/__init__.py b/glance/quota/__init__.py index d5e066b32d..40519928dc 100644 --- a/glance/quota/__init__.py +++ b/glance/quota/__init__.py @@ -16,6 +16,7 @@ import copy import six +import glance_store as store from oslo.config import cfg import glance.api.common @@ -57,9 +58,9 @@ def _calc_required_size(context, image, locations): for location in locations: size_from_backend = None try: - size_from_backend = glance.store.get_size_from_backend( - context, location['url']) - except (exception.UnknownScheme, exception.NotFound): + size_from_backend = store.get_size_from_backend( + location['url'], context=context) + except (store.UnknownScheme, store.NotFound): pass if size_from_backend: required_size = size_from_backend * len(locations) diff --git a/glance/scrubber.py b/glance/scrubber.py index 35ee877945..95977cfe82 100644 --- a/glance/scrubber.py +++ b/glance/scrubber.py @@ -53,6 +53,8 @@ scrubber_opts = [ 'clean up the files it uses for taking data. Only ' 'one server in your deployment should be designated ' 'the cleanup host.')), + cfg.BoolOpt('delayed_delete', default=False, + help=_('Turn on/off delayed delete.')), cfg.IntOpt('cleanup_scrubber_time', default=86400, help=_('Items must have a modified time that is older than ' 'this value in order to be candidates for cleanup.')) diff --git a/glance/store/__init__.py b/glance/store/__init__.py deleted file mode 100644 index d5a3051258..0000000000 --- a/glance/store/__init__.py +++ /dev/null @@ -1,384 +0,0 @@ -# Copyright 2010-2011 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -from oslo.config import cfg -import six - -from glance.common import exception -from glance.common import utils -import glance.context -import glance.domain.proxy -from glance.openstack.common import importutils -import glance.openstack.common.log as logging -from glance.store import location - -LOG = logging.getLogger(__name__) - -store_opts = [ - cfg.ListOpt('known_stores', - default=[ - 'glance.store.filesystem.Store', - 'glance.store.http.Store' - ], - help=_('List of which store classes and store class locations ' - 'are currently known to glance at startup.')), - cfg.StrOpt('default_store', default='file', - help=_("Default scheme to use to store image data. The " - "scheme must be registered by one of the stores " - "defined by the 'known_stores' config option.")), - cfg.BoolOpt('delayed_delete', default=False, - help=_('Turn on/off delayed delete.')), -] - -REGISTERED_STORES = set() -CONF = cfg.CONF -CONF.register_opts(store_opts) - -_ALL_STORES = [ - 'glance.store.filesystem.Store', - 'glance.store.http.Store', - 'glance.store.rbd.Store', - 'glance.store.s3.Store', - 'glance.store.swift.Store', - 'glance.store.sheepdog.Store', - 'glance.store.cinder.Store', - 'glance.store.gridfs.Store', - 'glance.store.vmware_datastore.Store' -] - - -class BackendException(Exception): - pass - - -class UnsupportedBackend(BackendException): - pass - - -class Indexable(object): - - """ - Wrapper that allows an iterator or filelike be treated as an indexable - data structure. This is required in the case where the return value from - Store.get() is passed to Store.add() when adding a Copy-From image to a - Store where the client library relies on eventlet GreenSockets, in which - case the data to be written is indexed over. - """ - - def __init__(self, wrapped, size): - """ - Initialize the object - - :param wrapped: the wrapped iterator or filelike. - :param size: the size of data available - """ - self.wrapped = wrapped - self.size = int(size) if size else (wrapped.len - if hasattr(wrapped, 'len') else 0) - self.cursor = 0 - self.chunk = None - - def __iter__(self): - """ - Delegate iteration to the wrapped instance. - """ - for self.chunk in self.wrapped: - yield self.chunk - - def __getitem__(self, i): - """ - Index into the next chunk (or previous chunk in the case where - the last data returned was not fully consumed). - - :param i: a slice-to-the-end - """ - start = i.start if isinstance(i, slice) else i - if start < self.cursor: - return self.chunk[(start - self.cursor):] - - self.chunk = self.another() - if self.chunk: - self.cursor += len(self.chunk) - - return self.chunk - - def another(self): - """Implemented by subclasses to return the next element""" - raise NotImplementedError - - def getvalue(self): - """ - Return entire string value... used in testing - """ - return self.wrapped.getvalue() - - def __len__(self): - """ - Length accessor. - """ - return self.size - - -def _register_stores(store_classes): - """ - Given a set of store names, add them to a globally available set - of store names. - """ - for store_cls in store_classes: - REGISTERED_STORES.add(store_cls.__module__.split('.')[2]) - # NOTE (spredzy): The actual class name is filesystem but in order - # to maintain backward compatibility we need to keep the 'file' store - # as a known store - if 'filesystem' in REGISTERED_STORES: - REGISTERED_STORES.add('file') - - -def _get_store_class(store_entry): - store_cls = None - try: - LOG.debug("Attempting to import store %s", store_entry) - store_cls = importutils.import_class(store_entry) - except exception.NotFound: - raise BackendException('Unable to load store. ' - 'Could not find a class named %s.' - % store_entry) - return store_cls - - -def create_stores(): - """ - Registers all store modules and all schemes - from the given config. Duplicates are not re-registered. - """ - store_count = 0 - store_classes = set() - for store_entry in set(CONF.known_stores + _ALL_STORES): - store_entry = store_entry.strip() - if not store_entry: - continue - store_cls = _get_store_class(store_entry) - try: - store_instance = store_cls() - except exception.BadStoreConfiguration as e: - if store_entry in CONF.known_stores: - LOG.warn(_("%s Skipping store driver.") % - utils.exception_to_str(e)) - continue - finally: - # NOTE(flaper87): To be removed in Juno - if store_entry not in CONF.known_stores: - LOG.deprecated(_("%s not found in `known_store`. " - "Stores need to be explicitly enabled in " - "the configuration file.") % store_entry) - - schemes = store_instance.get_schemes() - if not schemes: - raise BackendException('Unable to register store %s. ' - 'No schemes associated with it.' - % store_cls) - else: - if store_cls not in store_classes: - LOG.debug("Registering store %(cls)s with schemes " - "%(schemes)s", {'cls': store_cls, - 'schemes': schemes}) - store_classes.add(store_cls) - scheme_map = {} - for scheme in schemes: - loc_cls = store_instance.get_store_location_class() - scheme_map[scheme] = { - 'store_class': store_cls, - 'location_class': loc_cls, - } - location.register_scheme_map(scheme_map) - store_count += 1 - else: - LOG.debug("Store %s already registered", store_cls) - _register_stores(store_classes) - return store_count - - -def verify_default_store(): - scheme = cfg.CONF.default_store - context = glance.context.RequestContext() - try: - get_store_from_scheme(context, scheme, configure=False) - except exception.UnknownScheme: - msg = _("Store for scheme %s not found") % scheme - raise RuntimeError(msg) - - -def get_known_schemes(): - """Returns list of known schemes""" - return location.SCHEME_TO_CLS_MAP.keys() - - -def get_known_stores(): - """Returns list of known stores""" - return list(REGISTERED_STORES) - - -def get_store_from_scheme(context, scheme, loc=None, configure=True): - """ - Given a scheme, return the appropriate store object - for handling that scheme. - """ - if scheme not in location.SCHEME_TO_CLS_MAP: - raise exception.UnknownScheme(scheme=scheme) - scheme_info = location.SCHEME_TO_CLS_MAP[scheme] - store = scheme_info['store_class'](context, loc, configure) - return store - - -def get_store_from_uri(context, uri, loc=None): - """ - Given a URI, return the store object that would handle - operations on the URI. - - :param uri: URI to analyze - """ - scheme = uri[0:uri.find('/') - 1] - store = get_store_from_scheme(context, scheme, loc) - return store - - -def get_from_backend(context, uri, **kwargs): - """Yields chunks of data from backend specified by uri""" - - loc = location.get_location_from_uri(uri) - src_store = get_store_from_uri(context, uri, loc) - dest_store = kwargs.get('dest') - if dest_store is not None: - src_store.READ_CHUNKSIZE = dest_store.WRITE_CHUNKSIZE - try: - return src_store.get(loc) - except NotImplementedError: - raise exception.StoreGetNotSupported - - -def get_size_from_backend(context, uri): - """Retrieves image size from backend specified by uri""" - - loc = location.get_location_from_uri(uri) - store = get_store_from_uri(context, uri, loc) - - return store.get_size(loc) - - -def validate_location(context, uri): - loc = location.get_location_from_uri(uri) - store = get_store_from_uri(context, uri, loc) - store.validate_location(uri) - - -def delete_from_backend(context, uri, **kwargs): - """Removes chunks of data from backend specified by uri""" - loc = location.get_location_from_uri(uri) - store = get_store_from_uri(context, uri, loc) - - try: - return store.delete(loc) - except NotImplementedError: - raise exception.StoreDeleteNotSupported - - -def get_store_from_location(uri): - """ - Given a location (assumed to be a URL), attempt to determine - the store from the location. We use here a simple guess that - the scheme of the parsed URL is the store... - - :param uri: Location to check for the store - """ - loc = location.get_location_from_uri(uri) - return loc.store_name - - -def check_location_metadata(val, key=''): - if isinstance(val, dict): - for key in val: - check_location_metadata(val[key], key=key) - elif isinstance(val, list): - ndx = 0 - for v in val: - check_location_metadata(v, key='%s[%d]' % (key, ndx)) - ndx = ndx + 1 - elif not isinstance(val, six.text_type): - raise BackendException(_("The image metadata key %(key)s has an " - "invalid type of %(val)s. Only dict, list, " - "and unicode are supported.") % - {'key': key, - 'val': type(val)}) - - -def store_add_to_backend(image_id, data, size, store): - """ - A wrapper around a call to each stores add() method. This gives glance - a common place to check the output - - :param image_id: The image add to which data is added - :param data: The data to be stored - :param size: The length of the data in bytes - :param store: The store to which the data is being added - :return: The url location of the file, - the size amount of data, - the checksum of the data - the storage systems metadata dictionary for the location - """ - (location, size, checksum, metadata) = store.add(image_id, data, size) - if metadata is not None: - if not isinstance(metadata, dict): - msg = (_("The storage driver %(store)s returned invalid metadata " - "%(metadata)s. This must be a dictionary type") % - {'store': six.text_type(store), - 'metadata': six.text_type(metadata)}) - LOG.error(msg) - raise BackendException(msg) - try: - check_location_metadata(metadata) - except BackendException as e: - e_msg = (_("A bad metadata structure was returned from the " - "%(store)s storage driver: %(metadata)s. %(error)s.") % - {'store': six.text_type(store), - 'metadata': six.text_type(metadata), - 'error': utils.exception_to_str(e)}) - LOG.error(e_msg) - raise BackendException(e_msg) - return (location, size, checksum, metadata) - - -def add_to_backend(context, scheme, image_id, data, size): - store = get_store_from_scheme(context, scheme) - try: - return store_add_to_backend(image_id, data, size, store) - except NotImplementedError: - raise exception.StoreAddNotSupported - - -def set_acls(context, location_uri, public=False, read_tenants=None, - write_tenants=None): - if read_tenants is None: - read_tenants = [] - if write_tenants is None: - write_tenants = [] - - loc = location.get_location_from_uri(location_uri) - scheme = get_store_from_location(location_uri) - store = get_store_from_scheme(context, scheme, loc) - try: - store.set_acls(loc, public=public, read_tenants=read_tenants, - write_tenants=write_tenants) - except NotImplementedError: - LOG.debug("Skipping store.set_acls... not implemented.") diff --git a/glance/store/base.py b/glance/store/base.py deleted file mode 100644 index 16cc2daeb8..0000000000 --- a/glance/store/base.py +++ /dev/null @@ -1,176 +0,0 @@ -# Copyright 2011 OpenStack Foundation -# Copyright 2012 RedHat Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Base class for all storage backends""" - -from glance.common import exception -from glance.common import utils -from glance.openstack.common import importutils -import glance.openstack.common.log as logging -from glance.openstack.common import units - -LOG = logging.getLogger(__name__) - - -class Store(object): - - READ_CHUNKSIZE = 16 * units.Mi # 16M - WRITE_CHUNKSIZE = READ_CHUNKSIZE - - @staticmethod - def _unconfigured(*args, **kwargs): - raise exception.StoreNotConfigured - - def __init__(self, context=None, location=None, configure=True): - """ - Initialize the Store - """ - self.store_location_class = None - self.context = context - - if not configure: - self.add = Store._unconfigured - self.get = Store._unconfigured - self.get_size = Store._unconfigured - self.add_disabled = Store._unconfigured - self.delete = Store._unconfigured - self.set_acls = Store._unconfigured - return - - self.configure() - try: - self.configure_add() - except exception.BadStoreConfiguration as e: - self.add = self.add_disabled - msg = (_(u"Failed to configure store correctly: %s " - "Disabling add method.") % utils.exception_to_str(e)) - LOG.warn(msg) - - def configure(self): - """ - Configure the Store to use the stored configuration options - Any store that needs special configuration should implement - this method. - """ - pass - - def get_schemes(self): - """ - Returns a tuple of schemes which this store can handle. - """ - raise NotImplementedError - - def get_store_location_class(self): - """ - Returns the store location class that is used by this store. - """ - if not self.store_location_class: - class_name = "%s.StoreLocation" % (self.__module__) - LOG.debug("Late loading location class %s", class_name) - self.store_location_class = importutils.import_class(class_name) - return self.store_location_class - - def configure_add(self): - """ - This is like `configure` except that it's specifically for - configuring the store to accept objects. - - If the store was not able to successfully configure - itself, it should raise `exception.BadStoreConfiguration`. - """ - pass - - def validate_location(self, location): - """ - Takes a location and validates it for the presence - of any account references - """ - pass - - def get(self, location): - """ - Takes a `glance.store.location.Location` object that indicates - where to find the image file, and returns a tuple of generator - (for reading the image file) and image_size - - :param location `glance.store.location.Location` object, supplied - from glance.store.location.get_location_from_uri() - :raises `glance.exception.NotFound` if image does not exist - """ - raise NotImplementedError - - def get_size(self, location): - """ - Takes a `glance.store.location.Location` object that indicates - where to find the image file, and returns the size - - :param location `glance.store.location.Location` object, supplied - from glance.store.location.get_location_from_uri() - :raises `glance.exception.NotFound` if image does not exist - """ - raise NotImplementedError - - def add_disabled(self, *args, **kwargs): - """ - Add method that raises an exception because the Store was - not able to be configured properly and therefore the add() - method would error out. - """ - raise exception.StoreAddDisabled - - def add(self, image_id, image_file, image_size): - """ - Stores an image file with supplied identifier to the backend - storage system and returns a tuple containing information - about the stored image. - - :param image_id: The opaque image identifier - :param image_file: The image data to write, as a file-like object - :param image_size: The size of the image data to write, in bytes - - :retval tuple of URL in backing store, bytes written, checksum - and a dictionary with storage system specific information - :raises `glance.common.exception.Duplicate` if the image already - existed - """ - raise NotImplementedError - - def delete(self, location): - """ - Takes a `glance.store.location.Location` object that indicates - where to find the image file to delete - - :location `glance.store.location.Location` object, supplied - from glance.store.location.get_location_from_uri() - :raises `glance.exception.NotFound` if image does not exist - """ - raise NotImplementedError - - def set_acls(self, location, public=False, read_tenants=None, - write_tenants=None): - """ - Sets the read and write access control list for an image in the - backend store. - - :location `glance.store.location.Location` object, supplied - from glance.store.location.get_location_from_uri() - :public A boolean indicating whether the image should be public. - :read_tenants A list of tenant strings which should be granted - read access for an image. - :write_tenants A list of tenant strings which should be granted - write access for an image. - """ - raise NotImplementedError diff --git a/glance/store/cinder.py b/glance/store/cinder.py deleted file mode 100644 index b8bca3f4f4..0000000000 --- a/glance/store/cinder.py +++ /dev/null @@ -1,179 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Storage backend for Cinder""" - -from cinderclient import exceptions as cinder_exception -from cinderclient import service_catalog -from cinderclient.v2 import client as cinderclient - -from oslo.config import cfg - -from glance.common import exception -from glance.common import utils -import glance.openstack.common.log as logging -from glance.openstack.common import units -import glance.store.base -import glance.store.location - -LOG = logging.getLogger(__name__) - -cinder_opts = [ - cfg.StrOpt('cinder_catalog_info', - default='volume:cinder:publicURL', - help='Info to match when looking for cinder in the service ' - 'catalog. Format is: separated values of the form: ' - '::.'), - cfg.StrOpt('cinder_endpoint_template', - help='Override service catalog lookup with template for cinder ' - 'endpoint e.g. http://localhost:8776/v1/%(project_id)s.'), - cfg.StrOpt('os_region_name', - help='Region name of this node.'), - cfg.StrOpt('cinder_ca_certificates_file', - help='Location of CA certicates file to use for cinder client ' - 'requests.'), - cfg.IntOpt('cinder_http_retries', - default=3, - help='Number of cinderclient retries on failed http calls.'), - cfg.BoolOpt('cinder_api_insecure', - default=False, - help='Allow to perform insecure SSL requests to cinder.'), -] - -CONF = cfg.CONF -CONF.register_opts(cinder_opts) - - -def get_cinderclient(context): - if CONF.cinder_endpoint_template: - url = CONF.cinder_endpoint_template % context.to_dict() - else: - info = CONF.cinder_catalog_info - service_type, service_name, endpoint_type = info.split(':') - - # extract the region if set in configuration - if CONF.os_region_name: - attr = 'region' - filter_value = CONF.os_region_name - else: - attr = None - filter_value = None - - # FIXME: the cinderclient ServiceCatalog object is mis-named. - # It actually contains the entire access blob. - # Only needed parts of the service catalog are passed in, see - # nova/context.py. - compat_catalog = { - 'access': {'serviceCatalog': context.service_catalog or []}} - sc = service_catalog.ServiceCatalog(compat_catalog) - - url = sc.url_for(attr=attr, - filter_value=filter_value, - service_type=service_type, - service_name=service_name, - endpoint_type=endpoint_type) - - LOG.debug('Cinderclient connection created using URL: %s' % url) - - c = cinderclient.Client(context.user, - context.auth_tok, - project_id=context.tenant, - auth_url=url, - insecure=CONF.cinder_api_insecure, - retries=CONF.cinder_http_retries, - cacert=CONF.cinder_ca_certificates_file) - - # noauth extracts user_id:project_id from auth_token - c.client.auth_token = context.auth_tok or '%s:%s' % (context.user, - context.tenant) - c.client.management_url = url - return c - - -class StoreLocation(glance.store.location.StoreLocation): - - """Class describing a Cinder URI""" - - def process_specs(self): - self.scheme = self.specs.get('scheme', 'cinder') - self.volume_id = self.specs.get('volume_id') - - def get_uri(self): - return "cinder://%s" % self.volume_id - - def parse_uri(self, uri): - if not uri.startswith('cinder://'): - reason = _("URI must start with 'cinder://'") - LOG.info(reason) - raise exception.BadStoreUri(message=reason) - - self.scheme = 'cinder' - self.volume_id = uri[9:] - - if not utils.is_uuid_like(self.volume_id): - reason = _("URI contains invalid volume ID") - LOG.info(reason) - raise exception.BadStoreUri(message=reason) - - -class Store(glance.store.base.Store): - - """Cinder backend store adapter.""" - - EXAMPLE_URL = "cinder://volume-id" - - def get_schemes(self): - return ('cinder',) - - def configure_add(self): - """ - Configure the Store to use the stored configuration options - Any store that needs special configuration should implement - this method. If the store was not able to successfully configure - itself, it should raise `exception.BadStoreConfiguration` - """ - - if self.context is None: - reason = _("Cinder storage requires a context.") - raise exception.BadStoreConfiguration(store_name="cinder", - reason=reason) - if self.context.service_catalog is None: - reason = _("Cinder storage requires a service catalog.") - raise exception.BadStoreConfiguration(store_name="cinder", - reason=reason) - - def get_size(self, location): - """ - Takes a `glance.store.location.Location` object that indicates - where to find the image file and returns the image size - - :param location `glance.store.location.Location` object, supplied - from glance.store.location.get_location_from_uri() - :raises `glance.exception.NotFound` if image does not exist - :rtype int - """ - - loc = location.store_location - - try: - volume = get_cinderclient(self.context).volumes.get(loc.volume_id) - # GB unit convert to byte - return volume.size * units.Gi - except cinder_exception.NotFound as e: - reason = _("Failed to get image size due to " - "volume can not be found: %s") % self.volume_id - LOG.error(reason) - raise exception.NotFound(reason) - except Exception as e: - LOG.exception(_("Failed to get image size due to " - "internal error: %s") % utils.exception_to_str(e)) - return 0 diff --git a/glance/store/filesystem.py b/glance/store/filesystem.py deleted file mode 100644 index 8bb2ef8f9f..0000000000 --- a/glance/store/filesystem.py +++ /dev/null @@ -1,468 +0,0 @@ -# Copyright 2010 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -A simple filesystem-backed store -""" - -import errno -import hashlib -import os - -from oslo.config import cfg -import six.moves.urllib.parse as urlparse - -from glance.common import exception -from glance.common import utils -from glance.openstack.common import excutils -from glance.openstack.common import jsonutils -import glance.openstack.common.log as logging -from glance.openstack.common import processutils -from glance.openstack.common import units -import glance.store -import glance.store.base -import glance.store.location - -LOG = logging.getLogger(__name__) - -filesystem_opts = [ - cfg.StrOpt('filesystem_store_datadir', - help=_('Directory to which the Filesystem backend ' - 'store writes images.')), - cfg.MultiStrOpt('filesystem_store_datadirs', - help=_("List of directories and its priorities to which " - "the Filesystem backend store writes images.")), - cfg.StrOpt('filesystem_store_metadata_file', - help=_("The path to a file which contains the " - "metadata to be returned with any location " - "associated with this store. The file must " - "contain a valid JSON dict."))] - -CONF = cfg.CONF -CONF.register_opts(filesystem_opts) - - -class StoreLocation(glance.store.location.StoreLocation): - - """Class describing a Filesystem URI""" - - def process_specs(self): - self.scheme = self.specs.get('scheme', 'file') - self.path = self.specs.get('path') - - def get_uri(self): - return "file://%s" % self.path - - def parse_uri(self, uri): - """ - Parse URLs. This method fixes an issue where credentials specified - in the URL are interpreted differently in Python 2.6.1+ than prior - versions of Python. - """ - pieces = urlparse.urlparse(uri) - assert pieces.scheme in ('file', 'filesystem') - self.scheme = pieces.scheme - path = (pieces.netloc + pieces.path).strip() - if path == '': - reason = _("No path specified in URI") - LOG.info(reason) - raise exception.BadStoreUri(message=reason) - self.path = path - - -class ChunkedFile(object): - - """ - We send this back to the Glance API server as - something that can iterate over a large file - """ - - def __init__(self, filepath): - self.filepath = filepath - self.fp = open(self.filepath, 'rb') - - def __iter__(self): - """Return an iterator over the image file""" - try: - if self.fp: - while True: - chunk = self.fp.read(Store.READ_CHUNKSIZE) - if chunk: - yield chunk - else: - break - finally: - self.close() - - def close(self): - """Close the internal file pointer""" - if self.fp: - self.fp.close() - self.fp = None - - -class Store(glance.store.base.Store): - - READ_CHUNKSIZE = 64 * units.Ki - WRITE_CHUNKSIZE = READ_CHUNKSIZE - - def get_schemes(self): - return ('file', 'filesystem') - - def _check_write_permission(self, datadir): - """ - Checks if directory created to write image files has - write permission. - - :datadir is a directory path in which glance wites image files. - :raise BadStoreConfiguration exception if datadir is read-only. - """ - if not os.access(datadir, os.W_OK): - msg = (_("Permission to write in %s denied") % datadir) - LOG.exception(msg) - raise exception.BadStoreConfiguration( - store_name="filesystem", reason=msg) - - def _create_image_directories(self, directory_paths): - """ - Create directories to write image files if - it does not exist. - - :directory_paths is a list of directories belonging to glance store. - :raise BadStoreConfiguration exception if creating a directory fails. - """ - for datadir in directory_paths: - if os.path.exists(datadir): - self._check_write_permission(datadir) - else: - msg = _("Directory to write image files does not exist " - "(%s). Creating.") % datadir - LOG.info(msg) - try: - os.makedirs(datadir) - self._check_write_permission(datadir) - except (IOError, OSError): - if os.path.exists(datadir): - # NOTE(markwash): If the path now exists, some other - # process must have beat us in the race condition. - # But it doesn't hurt, so we can safely ignore - # the error. - self._check_write_permission(datadir) - continue - reason = _("Unable to create datadir: %s") % datadir - LOG.error(reason) - raise exception.BadStoreConfiguration( - store_name="filesystem", reason=reason) - - def configure_add(self): - """ - Configure the Store to use the stored configuration options - Any store that needs special configuration should implement - this method. If the store was not able to successfully configure - itself, it should raise `exception.BadStoreConfiguration` - """ - if not (CONF.filesystem_store_datadir - or CONF.filesystem_store_datadirs): - reason = (_("Specify at least 'filesystem_store_datadir' or " - "'filesystem_store_datadirs' option")) - LOG.error(reason) - raise exception.BadStoreConfiguration(store_name="filesystem", - reason=reason) - - if CONF.filesystem_store_datadir and CONF.filesystem_store_datadirs: - reason = (_("Specify either 'filesystem_store_datadir' or " - "'filesystem_store_datadirs' option")) - LOG.error(reason) - raise exception.BadStoreConfiguration(store_name="filesystem", - reason=reason) - - self.multiple_datadirs = False - directory_paths = set() - if CONF.filesystem_store_datadir: - self.datadir = CONF.filesystem_store_datadir - directory_paths.add(self.datadir) - else: - self.multiple_datadirs = True - self.priority_data_map = {} - for datadir in CONF.filesystem_store_datadirs: - (datadir_path, - priority) = self._get_datadir_path_and_priority(datadir) - self._check_directory_paths(datadir_path, directory_paths) - directory_paths.add(datadir_path) - self.priority_data_map.setdefault(int(priority), - []).append(datadir_path) - - self.priority_list = sorted(self.priority_data_map, - reverse=True) - - self._create_image_directories(directory_paths) - - def _check_directory_paths(self, datadir_path, directory_paths): - """ - Checks if directory_path is already present in directory_paths. - - :datadir_path is directory path. - :datadir_paths is set of all directory paths. - :raise BadStoreConfiguration exception if same directory path is - already present in directory_paths. - """ - if datadir_path in directory_paths: - msg = (_("Directory %(datadir_path)s specified " - "multiple times in filesystem_store_datadirs " - "option of filesystem configuration") % - {'datadir_path': datadir_path}) - LOG.exception(msg) - raise exception.BadStoreConfiguration( - store_name="filesystem", reason=msg) - - def _get_datadir_path_and_priority(self, datadir): - """ - Gets directory paths and its priority from - filesystem_store_datadirs option in glance-api.conf. - - :datadir is directory path with its priority. - :returns datadir_path as directory path - priority as priority associated with datadir_path - :raise BadStoreConfiguration exception if priority is invalid or - empty directory path is specified. - """ - priority = 0 - parts = map(lambda x: x.strip(), datadir.rsplit(":", 1)) - datadir_path = parts[0] - if len(parts) == 2 and parts[1]: - priority = parts[1] - if not priority.isdigit(): - msg = (_("Invalid priority value %(priority)s in " - "filesystem configuration") % {'priority': priority}) - LOG.exception(msg) - raise exception.BadStoreConfiguration( - store_name="filesystem", reason=msg) - - if not datadir_path: - msg = _("Invalid directory specified in filesystem configuration") - LOG.exception(msg) - raise exception.BadStoreConfiguration( - store_name="filesystem", reason=msg) - - return datadir_path, priority - - @staticmethod - def _resolve_location(location): - filepath = location.store_location.path - - if not os.path.exists(filepath): - raise exception.NotFound(_("Image file %s not found") % filepath) - - filesize = os.path.getsize(filepath) - return filepath, filesize - - def _get_metadata(self): - if CONF.filesystem_store_metadata_file is None: - return {} - - try: - with open(CONF.filesystem_store_metadata_file, 'r') as fptr: - metadata = jsonutils.load(fptr) - glance.store.check_location_metadata(metadata) - return metadata - except glance.store.BackendException as bee: - LOG.error(_('The JSON in the metadata file %(file)s could not be ' - 'used: %(error)s An empty dictionary will be ' - 'returned to the client.') % - {'file': CONF.filesystem_store_metadata_file, - 'error': utils.exception_to_str(bee)}) - return {} - except IOError as ioe: - LOG.error(_('The path for the metadata file %(file)s could not be ' - 'opened: %(error)s An empty dictionary will be ' - 'returned to the client.') % - {'file': CONF.filesystem_store_metadata_file, - 'error': utils.exception_to_str(ioe)}) - return {} - except Exception as ex: - LOG.exception(_('An error occurred processing the storage systems ' - 'meta data file: %s. An empty dictionary will be ' - 'returned to the client.') % - utils.exception_to_str(ex)) - return {} - - def get(self, location): - """ - Takes a `glance.store.location.Location` object that indicates - where to find the image file, and returns a tuple of generator - (for reading the image file) and image_size - - :param location `glance.store.location.Location` object, supplied - from glance.store.location.get_location_from_uri() - :raises `glance.exception.NotFound` if image does not exist - """ - filepath, filesize = self._resolve_location(location) - msg = "Found image at %s. Returning in ChunkedFile." % filepath - LOG.debug(msg) - return (ChunkedFile(filepath), filesize) - - def get_size(self, location): - """ - Takes a `glance.store.location.Location` object that indicates - where to find the image file and returns the image size - - :param location `glance.store.location.Location` object, supplied - from glance.store.location.get_location_from_uri() - :raises `glance.exception.NotFound` if image does not exist - :rtype int - """ - filepath, filesize = self._resolve_location(location) - msg = "Found image at %s." % filepath - LOG.debug(msg) - return filesize - - def delete(self, location): - """ - Takes a `glance.store.location.Location` object that indicates - where to find the image file to delete - - :location `glance.store.location.Location` object, supplied - from glance.store.location.get_location_from_uri() - - :raises NotFound if image does not exist - :raises Forbidden if cannot delete because of permissions - """ - loc = location.store_location - fn = loc.path - if os.path.exists(fn): - try: - LOG.debug("Deleting image at %(fn)s", {'fn': fn}) - os.unlink(fn) - except OSError: - raise exception.Forbidden(_("You cannot delete file %s") % fn) - else: - raise exception.NotFound(_("Image file %s does not exist") % fn) - - def _get_capacity_info(self, mount_point): - """Calculates total available space for given mount point. - - :mount_point is path of glance data directory - """ - - #Calculate total available space - df = processutils.execute("df", "-k", "-P", - mount_point)[0].strip("'\n'") - total_available_space = int(df.split('\n')[1].split()[3]) * units.Ki - - return max(0, total_available_space) - - def _find_best_datadir(self, image_size): - """Finds the best datadir by priority and free space. - - Traverse directories returning the first one that has sufficient - free space, in priority order. If two suitable directories have - the same priority, choose the one with the most free space - available. - :image_size size of image being uploaded. - :returns best_datadir as directory path of the best priority datadir. - :raises exception.StorageFull if there is no datadir in - self.priority_data_map that can accommodate the image. - """ - if not self.multiple_datadirs: - return self.datadir - - best_datadir = None - max_free_space = 0 - for priority in self.priority_list: - for datadir in self.priority_data_map.get(priority): - free_space = self._get_capacity_info(datadir) - if free_space >= image_size and free_space > max_free_space: - max_free_space = free_space - best_datadir = datadir - - # If datadir is found which can accommodate image and has maximum - # free space for the given priority then break the loop, - # else continue to lookup further. - if best_datadir: - break - else: - msg = (_("There is no enough disk space left on the image " - "storage media. requested=%s") % image_size) - LOG.exception(msg) - raise exception.StorageFull(message=msg) - - return best_datadir - - def add(self, image_id, image_file, image_size): - """ - Stores an image file with supplied identifier to the backend - storage system and returns a tuple containing information - about the stored image. - - :param image_id: The opaque image identifier - :param image_file: The image data to write, as a file-like object - :param image_size: The size of the image data to write, in bytes - - :retval tuple of URL in backing store, bytes written, checksum - and a dictionary with storage system specific information - :raises `glance.common.exception.Duplicate` if the image already - existed - - :note By default, the backend writes the image data to a file - `//`, where is the value of - the filesystem_store_datadir configuration option and - is the supplied image ID. - """ - datadir = self._find_best_datadir(image_size) - filepath = os.path.join(datadir, str(image_id)) - - if os.path.exists(filepath): - raise exception.Duplicate(_("Image file %s already exists!") - % filepath) - - checksum = hashlib.md5() - bytes_written = 0 - try: - with open(filepath, 'wb') as f: - for buf in utils.chunkreadable(image_file, - self.WRITE_CHUNKSIZE): - bytes_written += len(buf) - checksum.update(buf) - f.write(buf) - except IOError as e: - if e.errno != errno.EACCES: - self._delete_partial(filepath, image_id) - exceptions = {errno.EFBIG: exception.StorageFull(), - errno.ENOSPC: exception.StorageFull(), - errno.EACCES: exception.StorageWriteDenied()} - raise exceptions.get(e.errno, e) - except Exception: - with excutils.save_and_reraise_exception(): - self._delete_partial(filepath, image_id) - - checksum_hex = checksum.hexdigest() - metadata = self._get_metadata() - - LOG.debug("Wrote %(bytes_written)d bytes to %(filepath)s with " - "checksum %(checksum_hex)s", - {'bytes_written': bytes_written, - 'filepath': filepath, - 'checksum_hex': checksum_hex}) - return ('file://%s' % filepath, bytes_written, checksum_hex, metadata) - - @staticmethod - def _delete_partial(filepath, id): - try: - os.unlink(filepath) - except Exception as e: - msg = _('Unable to remove partial image data for image %(id)s: ' - '%(error)s') - LOG.error(msg % {'id': id, - 'error': utils.exception_to_str(e)}) diff --git a/glance/store/gridfs.py b/glance/store/gridfs.py deleted file mode 100644 index e0f51f4092..0000000000 --- a/glance/store/gridfs.py +++ /dev/null @@ -1,215 +0,0 @@ -# Copyright 2013 Red Hat, Inc -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Storage backend for GridFS""" -from __future__ import absolute_import - -from oslo.config import cfg -import six.moves.urllib.parse as urlparse - -from glance.common import exception -from glance.openstack.common import excutils -import glance.openstack.common.log as logging -import glance.store.base -import glance.store.location - -try: - import gridfs - import gridfs.errors - import pymongo - import pymongo.uri_parser as uri_parser -except ImportError: - pymongo = None - -LOG = logging.getLogger(__name__) - -gridfs_opts = [ - cfg.StrOpt('mongodb_store_uri', - help="Hostname or IP address of the instance to connect to, " - "or a mongodb URI, or a list of hostnames / mongodb URIs. " - "If host is an IPv6 literal it must be enclosed " - "in '[' and ']' characters following the RFC2732 " - "URL syntax (e.g. '[::1]' for localhost)."), - cfg.StrOpt('mongodb_store_db', help='Database to use.'), -] - -CONF = cfg.CONF -CONF.register_opts(gridfs_opts) - - -class StoreLocation(glance.store.location.StoreLocation): - """ - Class describing an gridfs URI: - - gridfs:// - - Connection information has been consciously omitted for - security reasons, since this location will be stored in glance's - database and can be queried from outside. - - Note(flaper87): Make connection info available if user wants so - by adding a new configuration parameter `mongdb_store_insecure`. - """ - - def get_uri(self): - return "gridfs://%s" % self.specs.get("image_id") - - def parse_uri(self, uri): - """ - This method should fix any issue with the passed URI. Right now, - it just sets image_id value in the specs dict. - - :param uri: Current set URI - """ - parsed = urlparse.urlparse(uri) - assert parsed.scheme in ('gridfs',) - self.specs["image_id"] = parsed.netloc - - -class Store(glance.store.base.Store): - """GridFS adapter""" - - EXAMPLE_URL = "gridfs://" - - def get_schemes(self): - return ('gridfs',) - - def configure_add(self): - """ - Configure the Store to use the stored configuration options - Any store that needs special configuration should implement - this method. If the store was not able to successfully configure - itself, it should raise `exception.BadStoreConfiguration` - """ - if pymongo is None: - msg = _("Missing dependencies: pymongo") - raise exception.BadStoreConfiguration(store_name="gridfs", - reason=msg) - - self.mongodb_uri = self._option_get('mongodb_store_uri') - - parsed = uri_parser.parse_uri(self.mongodb_uri) - self.mongodb_db = self._option_get('mongodb_store_db') or \ - parsed.get("database") - - self.mongodb = pymongo.MongoClient(self.mongodb_uri) - self.fs = gridfs.GridFS(self.mongodb[self.mongodb_db]) - - def _option_get(self, param): - result = getattr(CONF, param) - if not result: - reason = (_("Could not find %(param)s in configuration " - "options.") % {'param': param}) - LOG.debug(reason) - raise exception.BadStoreConfiguration(store_name="gridfs", - reason=reason) - return result - - def get(self, location): - """ - Takes a `glance.store.location.Location` object that indicates - where to find the image file, and returns a tuple of generator - (for reading the image file) and image_size - - :param location `glance.store.location.Location` object, supplied - from glance.store.location.get_location_from_uri() - :raises `glance.exception.NotFound` if image does not exist - """ - image = self._get_file(location) - return (image, image.length) - - def get_size(self, location): - """ - Takes a `glance.store.location.Location` object that indicates - where to find the image file, and returns the image_size (or 0 - if unavailable) - - :param location `glance.store.location.Location` object, supplied - from glance.store.location.get_location_from_uri() - """ - try: - key = self._get_file(location) - return key.length - except Exception: - return 0 - - def _get_file(self, location): - store_location = location - if isinstance(location, glance.store.location.Location): - store_location = location.store_location - try: - - parsed = urlparse.urlparse(store_location.get_uri()) - return self.fs.get(parsed.netloc) - except gridfs.errors.NoFile: - msg = ("Could not find %s image in GridFS" - % store_location.get_uri()) - LOG.debug(msg) - raise exception.NotFound(msg) - - def add(self, image_id, image_file, image_size): - """ - Stores an image file with supplied identifier to the backend - storage system and returns a tuple containing information - about the stored image. - - :param image_id: The opaque image identifier - :param image_file: The image data to write, as a file-like object - :param image_size: The size of the image data to write, in bytes - - :retval tuple of URL in backing store, bytes written, checksum - and a dictionary with storage system specific information - :raises `glance.common.exception.Duplicate` if the image already - existed - """ - loc = StoreLocation({'image_id': image_id}) - - if self.fs.exists(image_id): - raise exception.Duplicate(_("GridFS already has an image at " - "location %s") % loc.get_uri()) - - LOG.debug("Adding a new image to GridFS with id %(id)s and " - "size %(size)s" % {'id': image_id, - 'size': image_size}) - - try: - self.fs.put(image_file, _id=image_id) - image = self._get_file(loc) - except Exception: - # Note(zhiyan): clean up already received data when - # error occurs such as ImageSizeLimitExceeded exception. - with excutils.save_and_reraise_exception(): - self.fs.delete(image_id) - - LOG.debug("Uploaded image %(id)s, md5 %(md5)s, length %(length)s " - "to GridFS" % {'id': image._id, - 'md5': image.md5, - 'length': image.length}) - - return (loc.get_uri(), image.length, image.md5, {}) - - def delete(self, location): - """ - Takes a `glance.store.location.Location` object that indicates - where to find the image file to delete - - :location `glance.store.location.Location` object, supplied - from glance.store.location.get_location_from_uri() - - :raises NotFound if image does not exist - """ - image = self._get_file(location) - self.fs.delete(image._id) - LOG.debug("Deleted image %s from GridFS", image._id) diff --git a/glance/store/http.py b/glance/store/http.py deleted file mode 100644 index 5b47975161..0000000000 --- a/glance/store/http.py +++ /dev/null @@ -1,204 +0,0 @@ -# Copyright 2010 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import httplib -import socket - -import six.moves.urllib.parse as urlparse - -from glance.common import exception -import glance.openstack.common.log as logging -import glance.store.base -import glance.store.location - -LOG = logging.getLogger(__name__) - - -MAX_REDIRECTS = 5 - - -class StoreLocation(glance.store.location.StoreLocation): - - """Class describing an HTTP(S) URI""" - - def process_specs(self): - self.scheme = self.specs.get('scheme', 'http') - self.netloc = self.specs['netloc'] - self.user = self.specs.get('user') - self.password = self.specs.get('password') - self.path = self.specs.get('path') - - def _get_credstring(self): - if self.user: - return '%s:%s@' % (self.user, self.password) - return '' - - def get_uri(self): - return "%s://%s%s%s" % ( - self.scheme, - self._get_credstring(), - self.netloc, - self.path) - - def parse_uri(self, uri): - """ - Parse URLs. This method fixes an issue where credentials specified - in the URL are interpreted differently in Python 2.6.1+ than prior - versions of Python. - """ - pieces = urlparse.urlparse(uri) - assert pieces.scheme in ('https', 'http') - self.scheme = pieces.scheme - netloc = pieces.netloc - path = pieces.path - try: - if '@' in netloc: - creds, netloc = netloc.split('@') - else: - creds = None - except ValueError: - # Python 2.6.1 compat - # see lp659445 and Python issue7904 - if '@' in path: - creds, path = path.split('@') - else: - creds = None - if creds: - try: - self.user, self.password = creds.split(':') - except ValueError: - reason = _("Credentials are not well-formatted.") - LOG.info(reason) - raise exception.BadStoreUri(message=reason) - else: - self.user = None - if netloc == '': - reason = _("No address specified in HTTP URL") - LOG.info(reason) - raise exception.BadStoreUri(message=reason) - self.netloc = netloc - self.path = path - - -def http_response_iterator(conn, response, size): - """ - Return an iterator for a file-like object. - - :param conn: HTTP(S) Connection - :param response: httplib.HTTPResponse object - :param size: Chunk size to iterate with - """ - chunk = response.read(size) - while chunk: - yield chunk - chunk = response.read(size) - conn.close() - - -class Store(glance.store.base.Store): - - """An implementation of the HTTP(S) Backend Adapter""" - - def get(self, location): - """ - Takes a `glance.store.location.Location` object that indicates - where to find the image file, and returns a tuple of generator - (for reading the image file) and image_size - - :param location `glance.store.location.Location` object, supplied - from glance.store.location.get_location_from_uri() - """ - conn, resp, content_length = self._query(location, 'GET') - - iterator = http_response_iterator(conn, resp, self.READ_CHUNKSIZE) - - class ResponseIndexable(glance.store.Indexable): - def another(self): - try: - return self.wrapped.next() - except StopIteration: - return '' - - return (ResponseIndexable(iterator, content_length), content_length) - - def get_schemes(self): - return ('http', 'https') - - def get_size(self, location): - """ - Takes a `glance.store.location.Location` object that indicates - where to find the image file, and returns the size - - :param location `glance.store.location.Location` object, supplied - from glance.store.location.get_location_from_uri() - """ - try: - size = self._query(location, 'HEAD')[2] - except socket.error: - reason = _("The HTTP URL is invalid.") - LOG.info(reason) - raise exception.BadStoreUri(message=reason) - except Exception: - # NOTE(flaper87): Catch more granular exceptions, - # keeping this branch for backwards compatibility. - return 0 - return size - - def _query(self, location, verb, depth=0): - if depth > MAX_REDIRECTS: - reason = ("The HTTP URL exceeded %s maximum " - "redirects." % MAX_REDIRECTS) - LOG.debug(reason) - raise exception.MaxRedirectsExceeded(redirects=MAX_REDIRECTS) - loc = location.store_location - conn_class = self._get_conn_class(loc) - conn = conn_class(loc.netloc) - conn.request(verb, loc.path, "", {}) - resp = conn.getresponse() - - # Check for bad status codes - if resp.status >= 400: - if resp.status == httplib.NOT_FOUND: - reason = _("HTTP datastore could not find image at URI.") - LOG.debug(reason) - raise exception.NotFound(reason) - reason = _("HTTP URL returned a %s status code.") % resp.status - LOG.info(reason) - raise exception.BadStoreUri(message=reason) - - location_header = resp.getheader("location") - if location_header: - if resp.status not in (301, 302): - reason = (_("The HTTP URL attempted to redirect with an " - "invalid %s status code.") % resp.status) - LOG.info(reason) - raise exception.BadStoreUri(message=reason) - location_class = glance.store.location.Location - new_loc = location_class(location.store_name, - location.store_location.__class__, - uri=location_header, - image_id=location.image_id, - store_specs=location.store_specs) - return self._query(new_loc, verb, depth + 1) - content_length = int(resp.getheader('content-length', 0)) - return (conn, resp, content_length) - - def _get_conn_class(self, loc): - """ - Returns connection class for accessing the resource. Useful - for dependency injection and stubouts in testing... - """ - return {'http': httplib.HTTPConnection, - 'https': httplib.HTTPSConnection}[loc.scheme] diff --git a/glance/store/location.py b/glance/store/location.py deleted file mode 100644 index 5de3e4fdfe..0000000000 --- a/glance/store/location.py +++ /dev/null @@ -1,165 +0,0 @@ -# Copyright 2011 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -A class that describes the location of an image in Glance. - -In Glance, an image can either be **stored** in Glance, or it can be -**registered** in Glance but actually be stored somewhere else. - -We needed a class that could support the various ways that Glance -describes where exactly an image is stored. - -An image in Glance has two location properties: the image URI -and the image storage URI. - -The image URI is essentially the permalink identifier for the image. -It is displayed in the output of various Glance API calls and, -while read-only, is entirely user-facing. It shall **not** contain any -security credential information at all. The Glance image URI shall -be the host:port of that Glance API server along with /images/. - -The Glance storage URI is an internal URI structure that Glance -uses to maintain critical information about how to access the images -that it stores in its storage backends. It **may contain** security -credentials and is **not** user-facing. -""" - -import six.moves.urllib.parse as urlparse - -from glance.common import exception -import glance.openstack.common.log as logging - -LOG = logging.getLogger(__name__) - -SCHEME_TO_CLS_MAP = {} - - -def get_location_from_uri(uri): - """ - Given a URI, return a Location object that has had an appropriate - store parse the URI. - - :param uri: A URI that could come from the end-user in the Location - attribute/header - - Example URIs: - https://user:pass@example.com:80/images/some-id - http://images.oracle.com/123456 - swift://example.com/container/obj-id - swift://user:account:pass@authurl.com/container/obj-id - swift+http://user:account:pass@authurl.com/container/obj-id - s3://accesskey:secretkey@s3.amazonaws.com/bucket/key-id - s3+https://accesskey:secretkey@s3.amazonaws.com/bucket/key-id - file:///var/lib/glance/images/1 - cinder://volume-id - vsphere://server_host/folder/file_path?dcPath=dc_path&dsName=ds_name - """ - pieces = urlparse.urlparse(uri) - if pieces.scheme not in SCHEME_TO_CLS_MAP.keys(): - raise exception.UnknownScheme(scheme=pieces.scheme) - scheme_info = SCHEME_TO_CLS_MAP[pieces.scheme] - return Location(pieces.scheme, uri=uri, - store_location_class=scheme_info['location_class']) - - -def register_scheme_map(scheme_map): - """ - Given a mapping of 'scheme' to store_name, adds the mapping to the - known list of schemes if it does not already exist. - """ - for (k, v) in scheme_map.items(): - if k not in SCHEME_TO_CLS_MAP: - LOG.debug("Registering scheme %(k)s with %(v)s" % {'k': k, - 'v': v}) - SCHEME_TO_CLS_MAP[k] = v - - -class Location(object): - - """ - Class describing the location of an image that Glance knows about - """ - - def __init__(self, store_name, store_location_class, - uri=None, image_id=None, store_specs=None): - """ - Create a new Location object. - - :param store_name: The string identifier/scheme of the storage backend - :param store_location_class: The store location class to use - for this location instance. - :param image_id: The identifier of the image in whatever storage - backend is used. - :param uri: Optional URI to construct location from - :param store_specs: Dictionary of information about the location - of the image that is dependent on the backend - store - """ - self.store_name = store_name - self.image_id = image_id - self.store_specs = store_specs or {} - self.store_location = store_location_class(self.store_specs) - if uri: - self.store_location.parse_uri(uri) - - def get_store_uri(self): - """ - Returns the Glance image URI, which is the host:port of the API server - along with /images/ - """ - return self.store_location.get_uri() - - def get_uri(self): - return None - - -class StoreLocation(object): - - """ - Base class that must be implemented by each store - """ - - def __init__(self, store_specs): - self.specs = store_specs - if self.specs: - self.process_specs() - - def process_specs(self): - """ - Subclasses should implement any processing of the self.specs collection - such as storing credentials and possibly establishing connections. - """ - pass - - def get_uri(self): - """ - Subclasses should implement a method that returns an internal URI that, - when supplied to the StoreLocation instance, can be interpreted by the - StoreLocation's parse_uri() method. The URI returned from this method - shall never be public and only used internally within Glance, so it is - fine to encode credentials in this URI. - """ - raise NotImplementedError("StoreLocation subclass must implement " - "get_uri()") - - def parse_uri(self, uri): - """ - Subclasses should implement a method that accepts a string URI and - sets appropriate internal fields such that a call to get_uri() will - return a proper internal URI - """ - raise NotImplementedError("StoreLocation subclass must implement " - "parse_uri()") diff --git a/glance/store/rbd.py b/glance/store/rbd.py deleted file mode 100644 index c42a386645..0000000000 --- a/glance/store/rbd.py +++ /dev/null @@ -1,395 +0,0 @@ -# Copyright 2010-2011 Josh Durgin -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Storage backend for RBD - (RADOS (Reliable Autonomic Distributed Object Store) Block Device)""" -from __future__ import absolute_import -from __future__ import with_statement - -import hashlib -import math - -from oslo.config import cfg -import six -import six.moves.urllib.parse as urlparse -from six import text_type - -from glance.common import exception -from glance.common import utils -from glance import i18n -from glance.openstack.common import excutils -import glance.openstack.common.log as logging -from glance.openstack.common import units -import glance.store.base -import glance.store.location - -try: - import rados - import rbd -except ImportError: - rados = None - rbd = None - -DEFAULT_POOL = 'images' -DEFAULT_CONFFILE = '/etc/ceph/ceph.conf' -DEFAULT_USER = None # let librados decide based on the Ceph conf file -DEFAULT_CHUNKSIZE = 8 # in MiB -DEFAULT_SNAPNAME = 'snap' - -LOG = logging.getLogger(__name__) -_LI = i18n._LI - -rbd_opts = [ - cfg.IntOpt('rbd_store_chunk_size', default=DEFAULT_CHUNKSIZE, - help=_('RADOS images will be chunked into objects of this size ' - '(in megabytes). For best performance, this should be ' - 'a power of two.')), - cfg.StrOpt('rbd_store_pool', default=DEFAULT_POOL, - help=_('RADOS pool in which images are stored.')), - cfg.StrOpt('rbd_store_user', default=DEFAULT_USER, - help=_('RADOS user to authenticate as (only applicable if ' - 'using Cephx. If , a default will be chosen based ' - 'on the client. section in rbd_store_ceph_conf).')), - cfg.StrOpt('rbd_store_ceph_conf', default=DEFAULT_CONFFILE, - help=_('Ceph configuration file path. ' - 'If , librados will locate the default config. ' - 'If using cephx authentication, this file should ' - 'include a reference to the right keyring ' - 'in a client. section.')), -] - -CONF = cfg.CONF -CONF.register_opts(rbd_opts) - - -class StoreLocation(glance.store.location.StoreLocation): - """ - Class describing a RBD URI. This is of the form: - - rbd://image - - or - - rbd://fsid/pool/image/snapshot - """ - - def process_specs(self): - # convert to ascii since librbd doesn't handle unicode - for key, value in six.iteritems(self.specs): - self.specs[key] = str(value) - self.fsid = self.specs.get('fsid') - self.pool = self.specs.get('pool') - self.image = self.specs.get('image') - self.snapshot = self.specs.get('snapshot') - - def get_uri(self): - if self.fsid and self.pool and self.snapshot: - # ensure nothing contains / or any other url-unsafe character - safe_fsid = urlparse.quote(self.fsid, '') - safe_pool = urlparse.quote(self.pool, '') - safe_image = urlparse.quote(self.image, '') - safe_snapshot = urlparse.quote(self.snapshot, '') - return "rbd://%s/%s/%s/%s" % (safe_fsid, safe_pool, - safe_image, safe_snapshot) - else: - return "rbd://%s" % self.image - - def parse_uri(self, uri): - prefix = 'rbd://' - if not uri.startswith(prefix): - reason = _('URI must start with rbd://') - msg = _LI("Invalid URI: %s") % reason - LOG.info(msg) - raise exception.BadStoreUri(message=reason) - # convert to ascii since librbd doesn't handle unicode - try: - ascii_uri = str(uri) - except UnicodeError: - reason = _('URI contains non-ascii characters') - msg = _LI("Invalid URI: %s") % reason - LOG.info(msg) - raise exception.BadStoreUri(message=reason) - pieces = ascii_uri[len(prefix):].split('/') - if len(pieces) == 1: - self.fsid, self.pool, self.image, self.snapshot = \ - (None, None, pieces[0], None) - elif len(pieces) == 4: - self.fsid, self.pool, self.image, self.snapshot = \ - map(urlparse.unquote, pieces) - else: - reason = _('URI must have exactly 1 or 4 components') - msg = _LI("Invalid URI: %s") % reason - LOG.info(msg) - raise exception.BadStoreUri(message=reason) - if any(map(lambda p: p == '', pieces)): - reason = _('URI cannot contain empty components') - msg = _LI("Invalid URI: %s") % reason - LOG.info(msg) - raise exception.BadStoreUri(message=reason) - - -class ImageIterator(object): - """ - Reads data from an RBD image, one chunk at a time. - """ - - def __init__(self, name, store): - self.name = name - self.pool = store.pool - self.user = store.user - self.conf_file = store.conf_file - self.chunk_size = store.READ_CHUNKSIZE - - def __iter__(self): - try: - with rados.Rados(conffile=self.conf_file, - rados_id=self.user) as conn: - with conn.open_ioctx(self.pool) as ioctx: - with rbd.Image(ioctx, self.name) as image: - img_info = image.stat() - size = img_info['size'] - bytes_left = size - while bytes_left > 0: - length = min(self.chunk_size, bytes_left) - data = image.read(size - bytes_left, length) - bytes_left -= len(data) - yield data - raise StopIteration() - except rbd.ImageNotFound: - raise exception.NotFound( - _('RBD image %s does not exist') % self.name) - - -class Store(glance.store.base.Store): - """An implementation of the RBD backend adapter.""" - - EXAMPLE_URL = "rbd://///" - - def get_schemes(self): - return ('rbd',) - - def configure_add(self): - """ - Configure the Store to use the stored configuration options - Any store that needs special configuration should implement - this method. If the store was not able to successfully configure - itself, it should raise `exception.BadStoreConfiguration` - """ - try: - self.READ_CHUNKSIZE = CONF.rbd_store_chunk_size * units.Mi - self.WRITE_CHUNKSIZE = self.READ_CHUNKSIZE - - # these must not be unicode since they will be passed to a - # non-unicode-aware C library - self.pool = str(CONF.rbd_store_pool) - self.user = str(CONF.rbd_store_user) - self.conf_file = str(CONF.rbd_store_ceph_conf) - except cfg.ConfigFileValueError as e: - reason = (_("Error in store configuration: %s") % - utils.exception_to_str(e)) - LOG.error(reason) - raise exception.BadStoreConfiguration(store_name='rbd', - reason=reason) - - def get(self, location): - """ - Takes a `glance.store.location.Location` object that indicates - where to find the image file, and returns a tuple of generator - (for reading the image file) and image_size - - :param location `glance.store.location.Location` object, supplied - from glance.store.location.get_location_from_uri() - :raises `glance.exception.NotFound` if image does not exist - """ - loc = location.store_location - return (ImageIterator(loc.image, self), self.get_size(location)) - - def get_size(self, location): - """ - Takes a `glance.store.location.Location` object that indicates - where to find the image file, and returns the size - - :param location `glance.store.location.Location` object, supplied - from glance.store.location.get_location_from_uri() - :raises `glance.exception.NotFound` if image does not exist - """ - loc = location.store_location - with rados.Rados(conffile=self.conf_file, - rados_id=self.user) as conn: - with conn.open_ioctx(self.pool) as ioctx: - try: - with rbd.Image(ioctx, loc.image, - snapshot=loc.snapshot) as image: - img_info = image.stat() - return img_info['size'] - except rbd.ImageNotFound: - msg = 'RBD image %s does not exist' % loc.get_uri() - LOG.debug(msg) - raise exception.NotFound(msg) - - def _create_image(self, fsid, ioctx, image_name, size, order): - """ - Create an rbd image. If librbd supports it, - make it a cloneable snapshot, so that copy-on-write - volumes can be created from it. - - :param image_name Image's name - - :retval `glance.store.rbd.StoreLocation` object - """ - librbd = rbd.RBD() - if hasattr(rbd, 'RBD_FEATURE_LAYERING'): - librbd.create(ioctx, image_name, size, order, old_format=False, - features=rbd.RBD_FEATURE_LAYERING) - return StoreLocation({ - 'fsid': fsid, - 'pool': self.pool, - 'image': image_name, - 'snapshot': DEFAULT_SNAPNAME, - }) - else: - librbd.create(ioctx, image_name, size, order, old_format=True) - return StoreLocation({'image': image_name}) - - def _delete_image(self, image_name, snapshot_name=None): - """ - Delete RBD image and snapshot. - - :param image_name Image's name - :param snapshot_name Image snapshot's name - - :raises NotFound if image does not exist; - InUseByStore if image is in use or snapshot unprotect failed - """ - with rados.Rados(conffile=self.conf_file, rados_id=self.user) as conn: - with conn.open_ioctx(self.pool) as ioctx: - try: - # First remove snapshot. - if snapshot_name is not None: - with rbd.Image(ioctx, image_name) as image: - try: - image.unprotect_snap(snapshot_name) - except rbd.ImageBusy: - log_msg = ("snapshot %(image)s@%(snap)s " - "could not be unprotected because " - "it is in use") - LOG.debug(log_msg % - {'image': image_name, - 'snap': snapshot_name}) - raise exception.InUseByStore() - image.remove_snap(snapshot_name) - - # Then delete image. - rbd.RBD().remove(ioctx, image_name) - except rbd.ImageNotFound: - raise exception.NotFound( - _("RBD image %s does not exist") % image_name) - except rbd.ImageBusy: - log_msg = ("image %s could not be removed " - "because it is in use") - LOG.debug(log_msg % image_name) - raise exception.InUseByStore() - - def add(self, image_id, image_file, image_size): - """ - Stores an image file with supplied identifier to the backend - storage system and returns a tuple containing information - about the stored image. - - :param image_id: The opaque image identifier - :param image_file: The image data to write, as a file-like object - :param image_size: The size of the image data to write, in bytes - - :retval tuple of URL in backing store, bytes written, checksum - and a dictionary with storage system specific information - :raises `glance.common.exception.Duplicate` if the image already - existed - """ - checksum = hashlib.md5() - image_name = str(image_id) - with rados.Rados(conffile=self.conf_file, rados_id=self.user) as conn: - fsid = None - if hasattr(conn, 'get_fsid'): - fsid = conn.get_fsid() - with conn.open_ioctx(self.pool) as ioctx: - order = int(math.log(self.WRITE_CHUNKSIZE, 2)) - LOG.debug('creating image %(name)s with order %(order)d and ' - 'size %(size)d', - {'name': text_type(image_name), - 'order': order, - 'size': image_size}) - if image_size == 0: - LOG.warning(_("since image size is zero we will be doing " - "resize-before-write for each chunk which " - "will be considerably slower than normal")) - - try: - loc = self._create_image(fsid, ioctx, image_name, - image_size, order) - except rbd.ImageExists: - raise exception.Duplicate( - _('RBD image %s already exists') % image_id) - try: - with rbd.Image(ioctx, image_name) as image: - bytes_written = 0 - offset = 0 - chunks = utils.chunkreadable(image_file, - self.WRITE_CHUNKSIZE) - for chunk in chunks: - # If the image size provided is zero we need to do - # a resize for the amount we are writing. This will - # be slower so setting a higher chunk size may - # speed things up a bit. - if image_size == 0: - chunk_length = len(chunk) - length = offset + chunk_length - bytes_written += chunk_length - LOG.debug("resizing image to %s KiB" % - (length / units.Ki)) - image.resize(length) - LOG.debug("writing chunk at offset %s" % - (offset)) - offset += image.write(chunk, offset) - checksum.update(chunk) - if loc.snapshot: - image.create_snap(loc.snapshot) - image.protect_snap(loc.snapshot) - except Exception: - with excutils.save_and_reraise_exception(): - # Delete image if one was created - try: - self._delete_image(loc.image, loc.snapshot) - except exception.NotFound: - pass - - # Make sure we send back the image size whether provided or inferred. - if image_size == 0: - image_size = bytes_written - - return (loc.get_uri(), image_size, checksum.hexdigest(), {}) - - def delete(self, location): - """ - Takes a `glance.store.location.Location` object that indicates - where to find the image file to delete. - - :location `glance.store.location.Location` object, supplied - from glance.store.location.get_location_from_uri() - - :raises NotFound if image does not exist; - InUseByStore if image is in use or snapshot unprotect failed - """ - loc = location.store_location - self._delete_image(loc.image, loc.snapshot) diff --git a/glance/store/s3.py b/glance/store/s3.py deleted file mode 100644 index 591e3b4274..0000000000 --- a/glance/store/s3.py +++ /dev/null @@ -1,734 +0,0 @@ -# Copyright 2010 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Storage backend for S3 or Storage Servers that follow the S3 Protocol""" - -import hashlib -import httplib -import math -import re -import tempfile - -import boto.exception -import eventlet -from oslo.config import cfg -import six -import six.moves.urllib.parse as urlparse - -from glance.common import exception -from glance.common import utils -from glance import i18n -import glance.openstack.common.log as logging -from glance.openstack.common import units -import glance.store -import glance.store.base -import glance.store.location - -LOG = logging.getLogger(__name__) -_LE = i18n._LE -_LI = i18n._LI - -DEFAULT_LARGE_OBJECT_SIZE = 100 # 100M -DEFAULT_LARGE_OBJECT_CHUNK_SIZE = 10 # 10M -DEFAULT_LARGE_OBJECT_MIN_CHUNK_SIZE = 5 # 5M -DEFAULT_THREAD_POOLS = 10 # 10 pools - -s3_opts = [ - cfg.StrOpt('s3_store_host', - help=_('The host where the S3 server is listening.')), - cfg.StrOpt('s3_store_access_key', secret=True, - help=_('The S3 query token access key.')), - cfg.StrOpt('s3_store_secret_key', secret=True, - help=_('The S3 query token secret key.')), - cfg.StrOpt('s3_store_bucket', - help=_('The S3 bucket to be used to store the Glance data.')), - cfg.StrOpt('s3_store_object_buffer_dir', - help=_('The local directory where uploads will be staged ' - 'before they are transferred into S3.')), - cfg.BoolOpt('s3_store_create_bucket_on_put', default=False, - help=_('A boolean to determine if the S3 bucket should be ' - 'created on upload if it does not exist or if ' - 'an error should be returned to the user.')), - cfg.StrOpt('s3_store_bucket_url_format', default='subdomain', - help=_('The S3 calling format used to determine the bucket. ' - 'Either subdomain or path can be used.')), - cfg.IntOpt('s3_store_large_object_size', - default=DEFAULT_LARGE_OBJECT_SIZE, - help=_('What size, in MB, should S3 start chunking image files ' - 'and do a multipart upload in S3.')), - cfg.IntOpt('s3_store_large_object_chunk_size', - default=DEFAULT_LARGE_OBJECT_CHUNK_SIZE, - help=_('What multipart upload part size, in MB, should S3 use ' - 'when uploading parts. The size must be greater than or ' - 'equal to 5M.')), - cfg.IntOpt('s3_store_thread_pools', default=DEFAULT_THREAD_POOLS, - help=_('The number of thread pools to perform a multipart ' - 'upload in S3.')), -] - -CONF = cfg.CONF -CONF.register_opts(s3_opts) - - -class UploadPart: - - """ - The class for the upload part - """ - - def __init__(self, mpu, fp, partnum, chunks): - self.mpu = mpu - self.partnum = partnum - self.fp = fp - self.size = 0 - self.chunks = chunks - self.etag = {} # partnum -> etag - self.success = True - - -def run_upload(part): - """ - Upload the upload part into S3 and set returned etag and size - to its part info. - """ - pnum = part.partnum - bsize = part.chunks - LOG.info(_LI("Uploading upload part in S3 partnum=%(pnum)d, " - "size=%(bsize)d, key=%(key)s, UploadId=%(UploadId)s") % - {'pnum': pnum, - 'bsize': bsize, - 'key': part.mpu.key_name, - 'UploadId': part.mpu.id}) - - try: - key = part.mpu.upload_part_from_file(part.fp, - part_num=part.partnum, - size=bsize) - part.etag[part.partnum] = key.etag - part.size = key.size - except boto.exception.BotoServerError as e: - status = e.status - reason = e.reason - LOG.error(_LE("Failed to upload part in S3 partnum=%(pnum)d, " - "size=%(bsize)d, status=%(status)d, " - "reason=%(reason)s") % - {'pnum': pnum, - 'bsize': bsize, - 'status': status, - 'reason': reason}) - part.success = False - except Exception as e: - LOG.error(_LE("Failed to upload part in S3 partnum=%(pnum)d, " - "size=%(bsize)d due to internal error: %(err)s") % - {'pnum': pnum, - 'bsize': bsize, - 'err': utils.exception_to_str(e)}) - part.success = False - finally: - part.fp.close() - - -class StoreLocation(glance.store.location.StoreLocation): - - """ - Class describing an S3 URI. An S3 URI can look like any of - the following: - - s3://accesskey:secretkey@s3.amazonaws.com/bucket/key-id - s3+http://accesskey:secretkey@s3.amazonaws.com/bucket/key-id - s3+https://accesskey:secretkey@s3.amazonaws.com/bucket/key-id - - The s3+https:// URIs indicate there is an HTTPS s3service URL - """ - - def process_specs(self): - self.scheme = self.specs.get('scheme', 's3') - self.accesskey = self.specs.get('accesskey') - self.secretkey = self.specs.get('secretkey') - s3_host = self.specs.get('s3serviceurl') - self.bucket = self.specs.get('bucket') - self.key = self.specs.get('key') - - if s3_host.startswith('https://'): - self.scheme = 's3+https' - s3_host = s3_host[8:].strip('/') - elif s3_host.startswith('http://'): - s3_host = s3_host[7:].strip('/') - self.s3serviceurl = s3_host.strip('/') - - def _get_credstring(self): - if self.accesskey: - return '%s:%s@' % (self.accesskey, self.secretkey) - return '' - - def get_uri(self): - return "%s://%s%s/%s/%s" % ( - self.scheme, - self._get_credstring(), - self.s3serviceurl, - self.bucket, - self.key) - - def parse_uri(self, uri): - """ - Parse URLs. This method fixes an issue where credentials specified - in the URL are interpreted differently in Python 2.6.1+ than prior - versions of Python. - - Note that an Amazon AWS secret key can contain the forward slash, - which is entirely retarded, and breaks urlparse miserably. - This function works around that issue. - """ - # Make sure that URIs that contain multiple schemes, such as: - # s3://accesskey:secretkey@https://s3.amazonaws.com/bucket/key-id - # are immediately rejected. - if uri.count('://') != 1: - reason = _("URI cannot contain more than one occurrence " - "of a scheme. If you have specified a URI like " - "s3://accesskey:secretkey@" - "https://s3.amazonaws.com/bucket/key-id" - ", you need to change it to use the " - "s3+https:// scheme, like so: " - "s3+https://accesskey:secretkey@" - "s3.amazonaws.com/bucket/key-id") - LOG.info(_LI("Invalid store uri: %s") % reason) - raise exception.BadStoreUri(message=reason) - - pieces = urlparse.urlparse(uri) - assert pieces.scheme in ('s3', 's3+http', 's3+https') - self.scheme = pieces.scheme - path = pieces.path.strip('/') - netloc = pieces.netloc.strip('/') - entire_path = (netloc + '/' + path).strip('/') - - if '@' in uri: - creds, path = entire_path.split('@') - cred_parts = creds.split(':') - - try: - access_key = cred_parts[0] - secret_key = cred_parts[1] - # NOTE(jaypipes): Need to encode to UTF-8 here because of a - # bug in the HMAC library that boto uses. - # See: http://bugs.python.org/issue5285 - # See: http://trac.edgewall.org/ticket/8083 - access_key = access_key.encode('utf-8') - secret_key = secret_key.encode('utf-8') - self.accesskey = access_key - self.secretkey = secret_key - except IndexError: - reason = _("Badly formed S3 credentials") - LOG.info(reason) - raise exception.BadStoreUri(message=reason) - else: - self.accesskey = None - path = entire_path - try: - path_parts = path.split('/') - self.key = path_parts.pop() - self.bucket = path_parts.pop() - if path_parts: - self.s3serviceurl = '/'.join(path_parts).strip('/') - else: - reason = _("Badly formed S3 URI. Missing s3 service URL.") - raise exception.BadStoreUri(message=reason) - except IndexError: - reason = _("Badly formed S3 URI") - LOG.info(reason) - raise exception.BadStoreUri(message=reason) - - -class ChunkedFile(object): - - """ - We send this back to the Glance API server as - something that can iterate over a ``boto.s3.key.Key`` - """ - - def __init__(self, fp): - self.fp = fp - - def __iter__(self): - """Return an iterator over the image file""" - try: - if self.fp: - while True: - chunk = self.fp.read(Store.READ_CHUNKSIZE) - if chunk: - yield chunk - else: - break - finally: - self.close() - - def getvalue(self): - """Return entire string value... used in testing.""" - data = "" - self.len = 0 - for chunk in self: - read_bytes = len(chunk) - data = data + chunk - self.len = self.len + read_bytes - return data - - def close(self): - """Close the internal file pointer.""" - if self.fp: - self.fp.close() - self.fp = None - - -class Store(glance.store.base.Store): - """An implementation of the s3 adapter.""" - - READ_CHUNKSIZE = 64 * units.Ki - WRITE_CHUNKSIZE = READ_CHUNKSIZE - - EXAMPLE_URL = "s3://:@//" - - def get_schemes(self): - return ('s3', 's3+http', 's3+https') - - def configure_add(self): - """ - Configure the Store to use the stored configuration options - Any store that needs special configuration should implement - this method. If the store was not able to successfully configure - itself, it should raise `exception.BadStoreConfiguration` - """ - self.s3_host = self._option_get('s3_store_host') - access_key = self._option_get('s3_store_access_key') - secret_key = self._option_get('s3_store_secret_key') - # NOTE(jaypipes): Need to encode to UTF-8 here because of a - # bug in the HMAC library that boto uses. - # See: http://bugs.python.org/issue5285 - # See: http://trac.edgewall.org/ticket/8083 - self.access_key = access_key.encode('utf-8') - self.secret_key = secret_key.encode('utf-8') - self.bucket = self._option_get('s3_store_bucket') - - self.scheme = 's3' - if self.s3_host.startswith('https://'): - self.scheme = 's3+https' - self.full_s3_host = self.s3_host - elif self.s3_host.startswith('http://'): - self.full_s3_host = self.s3_host - else: # Defaults http - self.full_s3_host = 'http://' + self.s3_host - - self.s3_store_object_buffer_dir = CONF.s3_store_object_buffer_dir - - _s3_obj_size = CONF.s3_store_large_object_size - self.s3_store_large_object_size = _s3_obj_size * units.Mi - _s3_ck_size = CONF.s3_store_large_object_chunk_size - _s3_ck_min = DEFAULT_LARGE_OBJECT_MIN_CHUNK_SIZE - if _s3_ck_size < _s3_ck_min: - reason = (_("s3_store_large_object_chunk_size must be at " - "least %(_s3_ck_min)d MB. " - "You configured it as %(_s3_ck_size)d MB") % - {'_s3_ck_min': _s3_ck_min, - '_s3_ck_size': _s3_ck_size}) - LOG.error(reason) - raise exception.BadStoreConfiguration(store_name="s3", - reason=reason) - self.s3_store_large_object_chunk_size = _s3_ck_size * units.Mi - if CONF.s3_store_thread_pools <= 0: - reason = (_("s3_store_thread_pools must be a positive " - "integer. %s") % CONF.s3_store_thread_pools) - LOG.error(reason) - raise exception.BadStoreConfiguration(store_name="s3", - reason=reason) - - def _option_get(self, param): - result = getattr(CONF, param) - if not result: - reason = ("Could not find %(param)s in configuration " - "options." % {'param': param}) - LOG.debug(reason) - raise exception.BadStoreConfiguration(store_name="s3", - reason=reason) - return result - - def get(self, location): - """ - Takes a `glance.store.location.Location` object that indicates - where to find the image file, and returns a tuple of generator - (for reading the image file) and image_size - - :param location `glance.store.location.Location` object, supplied - from glance.store.location.get_location_from_uri() - :raises `glance.exception.NotFound` if image does not exist - """ - key = self._retrieve_key(location) - - key.BufferSize = self.READ_CHUNKSIZE - - class ChunkedIndexable(glance.store.Indexable): - def another(self): - return (self.wrapped.fp.read(self.READ_CHUNKSIZE) - if self.wrapped.fp else None) - - return (ChunkedIndexable(ChunkedFile(key), key.size), key.size) - - def get_size(self, location): - """ - Takes a `glance.store.location.Location` object that indicates - where to find the image file, and returns the image_size (or 0 - if unavailable) - - :param location `glance.store.location.Location` object, supplied - from glance.store.location.get_location_from_uri() - """ - try: - key = self._retrieve_key(location) - return key.size - except Exception: - return 0 - - def _retrieve_key(self, location): - loc = location.store_location - from boto.s3.connection import S3Connection - - s3_conn = S3Connection(loc.accesskey, loc.secretkey, - host=loc.s3serviceurl, - is_secure=(loc.scheme == 's3+https'), - calling_format=get_calling_format()) - bucket_obj = get_bucket(s3_conn, loc.bucket) - - key = get_key(bucket_obj, loc.key) - - msg = ("Retrieved image object from S3 using (s3_host=%(s3_host)s, " - "access_key=%(accesskey)s, bucket=%(bucket)s, " - "key=%(obj_name)s)" % ({'s3_host': loc.s3serviceurl, - 'accesskey': loc.accesskey, - 'bucket': loc.bucket, - 'obj_name': loc.key})) - LOG.debug(msg) - - return key - - def add(self, image_id, image_file, image_size): - """ - Stores an image file with supplied identifier to the backend - storage system and returns a tuple containing information - about the stored image. - - :param image_id: The opaque image identifier - :param image_file: The image data to write, as a file-like object - :param image_size: The size of the image data to write, in bytes - - :retval tuple of URL in backing store, bytes written, checksum - and a dictionary with storage system specific information - :raises `glance.common.exception.Duplicate` if the image already - existed - - S3 writes the image data using the scheme: - s3://:@// - where: - = ``s3_store_user`` - = ``s3_store_key`` - = ``s3_store_host`` - = ``s3_store_bucket`` - = The id of the image being added - """ - from boto.s3.connection import S3Connection - - loc = StoreLocation({'scheme': self.scheme, - 'bucket': self.bucket, - 'key': image_id, - 's3serviceurl': self.full_s3_host, - 'accesskey': self.access_key, - 'secretkey': self.secret_key}) - - s3_conn = S3Connection(loc.accesskey, loc.secretkey, - host=loc.s3serviceurl, - is_secure=(loc.scheme == 's3+https'), - calling_format=get_calling_format()) - - create_bucket_if_missing(self.bucket, s3_conn) - - bucket_obj = get_bucket(s3_conn, self.bucket) - obj_name = str(image_id) - - def _sanitize(uri): - return re.sub('//.*:.*@', - '//s3_store_secret_key:s3_store_access_key@', - uri) - - key = bucket_obj.get_key(obj_name) - if key and key.exists(): - raise exception.Duplicate(_("S3 already has an image at " - "location %s") % - _sanitize(loc.get_uri())) - - msg = ("Adding image object to S3 using (s3_host=%(s3_host)s, " - "access_key=%(access_key)s, bucket=%(bucket)s, " - "key=%(obj_name)s)" % ({'s3_host': self.s3_host, - 'access_key': self.access_key, - 'bucket': self.bucket, - 'obj_name': obj_name})) - LOG.debug(msg) - LOG.debug("Uploading an image file to S3 for %s" % - _sanitize(loc.get_uri())) - - if image_size < self.s3_store_large_object_size: - key = bucket_obj.new_key(obj_name) - - # We need to wrap image_file, which is a reference to the - # webob.Request.body_file, with a seekable file-like object, - # otherwise the call to set_contents_from_file() will die - # with an error about Input object has no method 'seek'. We - # might want to call webob.Request.make_body_seekable(), but - # unfortunately, that method copies the entire image into - # memory and results in LP Bug #818292 occurring. So, here - # we write temporary file in as memory-efficient manner as - # possible and then supply the temporary file to S3. We also - # take this opportunity to calculate the image checksum while - # writing the tempfile, so we don't need to call key.compute_md5() - - msg = ("Writing request body file to temporary file " - "for %s") % _sanitize(loc.get_uri()) - LOG.debug(msg) - - tmpdir = self.s3_store_object_buffer_dir - temp_file = tempfile.NamedTemporaryFile(dir=tmpdir) - checksum = hashlib.md5() - for chunk in utils.chunkreadable(image_file, self.WRITE_CHUNKSIZE): - checksum.update(chunk) - temp_file.write(chunk) - temp_file.flush() - - msg = ("Uploading temporary file to S3 " - "for %s") % _sanitize(loc.get_uri()) - LOG.debug(msg) - - # OK, now upload the data into the key - key.set_contents_from_file(open(temp_file.name, 'rb'), - replace=False) - size = key.size - checksum_hex = checksum.hexdigest() - - LOG.debug("Wrote %(size)d bytes to S3 key named %(obj_name)s " - "with checksum %(checksum_hex)s" % - {'size': size, - 'obj_name': obj_name, - 'checksum_hex': checksum_hex}) - - return (loc.get_uri(), size, checksum_hex, {}) - else: - checksum = hashlib.md5() - parts = int(math.ceil(float(image_size) / - float(self.s3_store_large_object_chunk_size))) - threads = parts - - pool_size = CONF.s3_store_thread_pools - pool = eventlet.greenpool.GreenPool(size=pool_size) - mpu = bucket_obj.initiate_multipart_upload(obj_name) - LOG.debug("Multipart initiate key=%(obj_name)s, " - "UploadId=%(UploadId)s" % - {'obj_name': obj_name, - 'UploadId': mpu.id}) - cstart = 0 - plist = [] - - it = utils.chunkreadable(image_file, - self.s3_store_large_object_chunk_size) - - for p in range(threads): - chunk = next(it) - clen = len(chunk) - checksum.update(chunk) - fp = six.BytesIO(chunk) - fp.seek(0) - part = UploadPart(mpu, fp, cstart + 1, clen) - pool.spawn_n(run_upload, part) - plist.append(part) - cstart += 1 - - pedict = {} - total_size = 0 - pool.waitall() - - for part in plist: - pedict.update(part.etag) - total_size += part.size - - success = True - for part in plist: - if not part.success: - success = False - - if success: - # Complete - xml = get_mpu_xml(pedict) - bucket_obj.complete_multipart_upload(obj_name, - mpu.id, - xml) - checksum_hex = checksum.hexdigest() - LOG.info(_LI("Multipart complete key=%(obj_name)s " - "UploadId=%(UploadId)s " - "Wrote %(total_size)d bytes to S3 key" - "named %(obj_name)s " - "with checksum %(checksum_hex)s") % - {'obj_name': obj_name, - 'UploadId': mpu.id, - 'total_size': total_size, - 'obj_name': obj_name, - 'checksum_hex': checksum_hex}) - return (loc.get_uri(), total_size, checksum_hex, {}) - else: - # Abort - bucket_obj.cancel_multipart_upload(obj_name, mpu.id) - LOG.error(_LE("Some parts failed to upload to S3. " - "Aborted the object key=%(obj_name)s") % - {'obj_name': obj_name}) - msg = (_("Failed to add image object to S3. " - "key=%(obj_name)s") % {'obj_name': obj_name}) - raise glance.store.BackendException(msg) - - def delete(self, location): - """ - Takes a `glance.store.location.Location` object that indicates - where to find the image file to delete - - :location `glance.store.location.Location` object, supplied - from glance.store.location.get_location_from_uri() - - :raises NotFound if image does not exist - """ - loc = location.store_location - from boto.s3.connection import S3Connection - s3_conn = S3Connection(loc.accesskey, loc.secretkey, - host=loc.s3serviceurl, - is_secure=(loc.scheme == 's3+https'), - calling_format=get_calling_format()) - bucket_obj = get_bucket(s3_conn, loc.bucket) - - # Close the key when we're through. - key = get_key(bucket_obj, loc.key) - - msg = ("Deleting image object from S3 using (s3_host=%(s3_host)s, " - "access_key=%(accesskey)s, bucket=%(bucket)s, " - "key=%(obj_name)s)" % ({'s3_host': loc.s3serviceurl, - 'accesskey': loc.accesskey, - 'bucket': loc.bucket, - 'obj_name': loc.key})) - LOG.debug(msg) - - return key.delete() - - -def get_bucket(conn, bucket_id): - """ - Get a bucket from an s3 connection - - :param conn: The ``boto.s3.connection.S3Connection`` - :param bucket_id: ID of the bucket to fetch - :raises ``glance.exception.NotFound`` if bucket is not found. - """ - - bucket = conn.get_bucket(bucket_id) - if not bucket: - msg = "Could not find bucket with ID %s" % bucket_id - LOG.debug(msg) - raise exception.NotFound(msg) - - return bucket - - -def get_s3_location(s3_host): - from boto.s3.connection import Location - locations = { - 's3.amazonaws.com': Location.DEFAULT, - 's3-eu-west-1.amazonaws.com': Location.EU, - 's3-us-west-1.amazonaws.com': Location.USWest, - 's3-ap-southeast-1.amazonaws.com': Location.APSoutheast, - 's3-ap-northeast-1.amazonaws.com': Location.APNortheast, - } - # strip off scheme and port if present - key = re.sub('^(https?://)?(?P[^:]+)(:[0-9]+)?$', - '\g', - s3_host) - return locations.get(key, Location.DEFAULT) - - -def create_bucket_if_missing(bucket, s3_conn): - """ - Creates a missing bucket in S3 if the - ``s3_store_create_bucket_on_put`` option is set. - - :param bucket: Name of bucket to create - :param s3_conn: Connection to S3 - """ - from boto.exception import S3ResponseError - try: - s3_conn.get_bucket(bucket) - except S3ResponseError as e: - if e.status == httplib.NOT_FOUND: - if CONF.s3_store_create_bucket_on_put: - location = get_s3_location(CONF.s3_store_host) - try: - s3_conn.create_bucket(bucket, location=location) - except S3ResponseError as e: - msg = (_("Failed to add bucket to S3.\n" - "Got error from S3: %s") % - utils.exception_to_str(e)) - raise glance.store.BackendException(msg) - else: - msg = (_("The bucket %(bucket)s does not exist in " - "S3. Please set the " - "s3_store_create_bucket_on_put option " - "to add bucket to S3 automatically.") - % {'bucket': bucket}) - raise glance.store.BackendException(msg) - - -def get_key(bucket, obj): - """ - Get a key from a bucket - - :param bucket: The ``boto.s3.Bucket`` - :param obj: Object to get the key for - :raises ``glance.exception.NotFound`` if key is not found. - """ - - key = bucket.get_key(obj) - if not key or not key.exists(): - msg = ("Could not find key %(obj)s in bucket %(bucket)s" % - {'obj': obj, 'bucket': bucket}) - LOG.debug(msg) - raise exception.NotFound(msg) - return key - - -def get_calling_format(bucket_format=None): - import boto.s3.connection - if bucket_format is None: - bucket_format = CONF.s3_store_bucket_url_format - if bucket_format.lower() == 'path': - return boto.s3.connection.OrdinaryCallingFormat() - else: - return boto.s3.connection.SubdomainCallingFormat() - - -def get_mpu_xml(pedict): - xml = '\n' - for pnum, etag in pedict.iteritems(): - xml += ' \n' - xml += ' %d\n' % pnum - xml += ' %s\n' % etag - xml += ' \n' - xml += '' - return xml diff --git a/glance/store/sheepdog.py b/glance/store/sheepdog.py deleted file mode 100644 index f99956bd4d..0000000000 --- a/glance/store/sheepdog.py +++ /dev/null @@ -1,319 +0,0 @@ -# Copyright 2013 Taobao Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Storage backend for Sheepdog storage system""" - -import hashlib - -from oslo.config import cfg - -from glance.common import exception -from glance.common import utils -from glance.openstack.common import excutils -import glance.openstack.common.log as logging -from glance.openstack.common import processutils -from glance.openstack.common import units -import glance.store -import glance.store.base -import glance.store.location - - -LOG = logging.getLogger(__name__) - -DEFAULT_ADDR = '127.0.0.1' -DEFAULT_PORT = 7000 -DEFAULT_CHUNKSIZE = 64 # in MiB - -LOG = logging.getLogger(__name__) - -sheepdog_opts = [ - cfg.IntOpt('sheepdog_store_chunk_size', default=DEFAULT_CHUNKSIZE, - help=_('Images will be chunked into objects of this size ' - '(in megabytes). For best performance, this should be ' - 'a power of two.')), - cfg.IntOpt('sheepdog_store_port', default=DEFAULT_PORT, - help=_('Port of sheep daemon.')), - cfg.StrOpt('sheepdog_store_address', default=DEFAULT_ADDR, - help=_('IP address of sheep daemon.')) -] - -CONF = cfg.CONF -CONF.register_opts(sheepdog_opts) - - -class SheepdogImage: - """Class describing an image stored in Sheepdog storage.""" - - def __init__(self, addr, port, name, chunk_size): - self.addr = addr - self.port = port - self.name = name - self.chunk_size = chunk_size - - def _run_command(self, command, data, *params): - cmd = ["collie", "vdi"] - cmd.extend(command) - cmd.extend(["-a", self.addr, "-p", self.port, self.name]) - cmd.extend(params) - - try: - return processutils.execute(*cmd, process_input=data)[0] - except (processutils.ProcessExecutionError, OSError) as exc: - LOG.error(exc) - raise glance.store.BackendException(exc) - - def get_size(self): - """ - Return the size of the this iamge - - Sheepdog Usage: collie vdi list -r -a address -p port image - """ - out = self._run_command(["list", "-r"], None) - return long(out.split(' ')[3]) - - def read(self, offset, count): - """ - Read up to 'count' bytes from this image starting at 'offset' and - return the data. - - Sheepdog Usage: collie vdi read -a address -p port image offset len - """ - return self._run_command(["read"], None, str(offset), str(count)) - - def write(self, data, offset, count): - """ - Write up to 'count' bytes from the data to this image starting at - 'offset' - - Sheepdog Usage: collie vdi write -a address -p port image offset len - """ - self._run_command(["write"], data, str(offset), str(count)) - - def create(self, size): - """ - Create this image in the Sheepdog cluster with size 'size'. - - Sheepdog Usage: collie vdi create -a address -p port image size - """ - self._run_command(["create"], None, str(size)) - - def delete(self): - """ - Delete this image in the Sheepdog cluster - - Sheepdog Usage: collie vdi delete -a address -p port image - """ - self._run_command(["delete"], None) - - def exist(self): - """ - Check if this image exists in the Sheepdog cluster via 'list' command - - Sheepdog Usage: collie vdi list -r -a address -p port image - """ - out = self._run_command(["list", "-r"], None) - if not out: - return False - else: - return True - - -class StoreLocation(glance.store.location.StoreLocation): - """ - Class describing a Sheepdog URI. This is of the form: - - sheepdog://image-id - - """ - - def process_specs(self): - self.image = self.specs.get('image') - - def get_uri(self): - return "sheepdog://%s" % self.image - - def parse_uri(self, uri): - valid_schema = 'sheepdog://' - if not uri.startswith(valid_schema): - reason = _("URI must start with '%s://'") % valid_schema - raise exception.BadStoreUri(message=reason) - self.image = uri[len(valid_schema):] - if not utils.is_uuid_like(self.image): - reason = _("URI must contains well-formated image id") - raise exception.BadStoreUri(message=reason) - - -class ImageIterator(object): - """ - Reads data from an Sheepdog image, one chunk at a time. - """ - - def __init__(self, image): - self.image = image - - def __iter__(self): - image = self.image - total = left = image.get_size() - while left > 0: - length = min(image.chunk_size, left) - data = image.read(total - left, length) - left -= len(data) - yield data - raise StopIteration() - - -class Store(glance.store.base.Store): - """Sheepdog backend adapter.""" - - EXAMPLE_URL = "sheepdog://image" - - def get_schemes(self): - return ('sheepdog',) - - def configure(self): - """ - Configure the Store to use the stored configuration options - Any store that needs special configuration should implement - this method. If the store was not able to successfully configure - itself, it should raise `exception.BadStoreConfiguration` - """ - - try: - self.READ_CHUNKSIZE = CONF.sheepdog_store_chunk_size * units.Mi - self.WRITE_CHUNKSIZE = self.READ_CHUNKSIZE - self.addr = CONF.sheepdog_store_address.strip() - self.port = CONF.sheepdog_store_port - except cfg.ConfigFileValueError as e: - reason = (_("Error in store configuration: %s") % - utils.exception_to_str(e)) - LOG.error(reason) - raise exception.BadStoreConfiguration(store_name='sheepdog', - reason=reason) - - if ' ' in self.addr: - reason = (_("Invalid address configuration of sheepdog store: %s") - % self.addr) - LOG.error(reason) - raise exception.BadStoreConfiguration(store_name='sheepdog', - reason=reason) - - try: - cmd = ["collie", "vdi", "list", "-a", self.addr, "-p", self.port] - processutils.execute(*cmd) - except Exception as e: - reason = (_("Error in store configuration: %s") % - utils.exception_to_str(e)) - LOG.error(reason) - raise exception.BadStoreConfiguration(store_name='sheepdog', - reason=reason) - - def get(self, location): - """ - Takes a `glance.store.location.Location` object that indicates - where to find the image file, and returns a generator for reading - the image file - - :param location `glance.store.location.Location` object, supplied - from glance.store.location.get_location_from_uri() - :raises `glance.exception.NotFound` if image does not exist - """ - - loc = location.store_location - image = SheepdogImage(self.addr, self.port, loc.image, - self.READ_CHUNKSIZE) - if not image.exist(): - raise exception.NotFound(_("Sheepdog image %s does not exist") - % image.name) - return (ImageIterator(image), image.get_size()) - - def get_size(self, location): - """ - Takes a `glance.store.location.Location` object that indicates - where to find the image file and returns the image size - - :param location `glance.store.location.Location` object, supplied - from glance.store.location.get_location_from_uri() - :raises `glance.exception.NotFound` if image does not exist - :rtype int - """ - - loc = location.store_location - image = SheepdogImage(self.addr, self.port, loc.image, - self.READ_CHUNKSIZE) - if not image.exist(): - raise exception.NotFound(_("Sheepdog image %s does not exist") - % image.name) - return image.get_size() - - def add(self, image_id, image_file, image_size): - """ - Stores an image file with supplied identifier to the backend - storage system and returns a tuple containing information - about the stored image. - - :param image_id: The opaque image identifier - :param image_file: The image data to write, as a file-like object - :param image_size: The size of the image data to write, in bytes - - :retval tuple of URL in backing store, bytes written, and checksum - :raises `glance.common.exception.Duplicate` if the image already - existed - """ - - image = SheepdogImage(self.addr, self.port, image_id, - self.WRITE_CHUNKSIZE) - if image.exist(): - raise exception.Duplicate(_("Sheepdog image %s already exists") - % image_id) - - location = StoreLocation({'image': image_id}) - checksum = hashlib.md5() - - image.create(image_size) - - try: - total = left = image_size - while left > 0: - length = min(self.WRITE_CHUNKSIZE, left) - data = image_file.read(length) - image.write(data, total - left, length) - left -= length - checksum.update(data) - except Exception: - # Note(zhiyan): clean up already received data when - # error occurs such as ImageSizeLimitExceeded exception. - with excutils.save_and_reraise_exception(): - image.delete() - - return (location.get_uri(), image_size, checksum.hexdigest(), {}) - - def delete(self, location): - """ - Takes a `glance.store.location.Location` object that indicates - where to find the image file to delete - - :location `glance.store.location.Location` object, supplied - from glance.store.location.get_location_from_uri() - - :raises NotFound if image does not exist - """ - - loc = location.store_location - image = SheepdogImage(self.addr, self.port, loc.image, - self.WRITe_CHUNKSIZE) - if not image.exist(): - raise exception.NotFound(_("Sheepdog image %s does not exist") % - loc.image) - image.delete() diff --git a/glance/store/swift.py b/glance/store/swift.py deleted file mode 100644 index 92b9f65e68..0000000000 --- a/glance/store/swift.py +++ /dev/null @@ -1,826 +0,0 @@ -# Copyright 2010-2011 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Storage backend for SWIFT""" - -from __future__ import absolute_import - -import hashlib -import httplib -import math - -from oslo.config import cfg -import six.moves.urllib.parse as urlparse -import urllib - -from glance.common import auth -from glance.common import exception -from glance.common import swift_store_utils -from glance.common import utils -from glance import i18n -from glance.openstack.common import excutils -import glance.openstack.common.log as logging -from glance.openstack.common import units -import glance.store -import glance.store.base -import glance.store.location - -try: - import swiftclient -except ImportError: - pass - -LOG = logging.getLogger(__name__) -_LI = i18n._LI - -DEFAULT_CONTAINER = 'glance' -DEFAULT_LARGE_OBJECT_SIZE = 5 * 1024 # 5GB -DEFAULT_LARGE_OBJECT_CHUNK_SIZE = 200 # 200M -ONE_MB = 1000 * 1024 - -swift_opts = [ - cfg.BoolOpt('swift_enable_snet', default=False, - help=_('Whether to use ServiceNET to communicate with the ' - 'Swift storage servers.')), - cfg.StrOpt('swift_store_auth_version', default='2', - help=_('Version of the authentication service to use. ' - 'Valid versions are 2 for keystone and 1 for swauth ' - 'and rackspace. (deprecated)')), - cfg.BoolOpt('swift_store_auth_insecure', default=False, - help=_('If True, swiftclient won\'t check for a valid SSL ' - 'certificate when authenticating.')), - cfg.StrOpt('swift_store_region', - help=_('The region of the swift endpoint to be used for ' - 'single tenant. This setting is only necessary if the ' - 'tenant has multiple swift endpoints.')), - cfg.StrOpt('swift_store_endpoint_type', default='publicURL', - help=_('A string giving the endpoint type of the swift ' - 'service to use (publicURL, adminURL or internalURL). ' - 'This setting is only used if swift_store_auth_version ' - 'is 2.')), - cfg.StrOpt('swift_store_service_type', default='object-store', - help=_('A string giving the service type of the swift service ' - 'to use. This setting is only used if ' - 'swift_store_auth_version is 2.')), - cfg.StrOpt('swift_store_container', - default=DEFAULT_CONTAINER, - help=_('Container within the account that the account should ' - 'use for storing images in Swift.')), - cfg.IntOpt('swift_store_large_object_size', - default=DEFAULT_LARGE_OBJECT_SIZE, - help=_('The size, in MB, that Glance will start chunking image ' - 'files and do a large object manifest in Swift.')), - cfg.IntOpt('swift_store_large_object_chunk_size', - default=DEFAULT_LARGE_OBJECT_CHUNK_SIZE, - help=_('The amount of data written to a temporary disk buffer ' - 'during the process of chunking the image file.')), - cfg.BoolOpt('swift_store_create_container_on_put', default=False, - help=_('A boolean value that determines if we create the ' - 'container if it does not exist.')), - cfg.BoolOpt('swift_store_multi_tenant', default=False, - help=_('If set to True, enables multi-tenant storage ' - 'mode which causes Glance images to be stored in ' - 'tenant specific Swift accounts.')), - cfg.ListOpt('swift_store_admin_tenants', default=[], - help=_('A list of tenants that will be granted read/write ' - 'access on all Swift containers created by Glance in ' - 'multi-tenant mode.')), - cfg.BoolOpt('swift_store_ssl_compression', default=True, - help=_('If set to False, disables SSL layer compression of ' - 'https swift requests. Setting to False may improve ' - 'performance for images which are already in a ' - 'compressed format, eg qcow2.')), - cfg.IntOpt('swift_store_retry_get_count', default=0, - help=_('The number of times a Swift download will be retried ' - 'before the request fails.')) -] - -CONF = cfg.CONF -CONF.register_opts(swift_opts) - -SWIFT_STORE_REF_PARAMS = swift_store_utils.SwiftParams().params - - -def swift_retry_iter(resp_iter, length, store, location): - length = length if length else (resp_iter.len - if hasattr(resp_iter, 'len') else 0) - retries = 0 - bytes_read = 0 - - while retries <= CONF.swift_store_retry_get_count: - try: - for chunk in resp_iter: - yield chunk - bytes_read += len(chunk) - except swiftclient.ClientException as e: - LOG.warn(_("Swift exception raised %s") % - utils.exception_to_str(e)) - - if bytes_read != length: - if retries == CONF.swift_store_retry_get_count: - # terminate silently and let higher level decide - LOG.error(_("Stopping Swift retries after %d " - "attempts") % retries) - break - else: - retries += 1 - LOG.info(_("Retrying Swift connection " - "(%(retries)d/%(max_retries)d) with " - "range=%(start)d-%(end)d") % - {'retries': retries, - 'max_retries': CONF.swift_store_retry_get_count, - 'start': bytes_read, - 'end': length}) - (resp_headers, resp_iter) = store._get_object(location, None, - bytes_read) - else: - break - - -class StoreLocation(glance.store.location.StoreLocation): - - """ - Class describing a Swift URI. A Swift URI can look like any of - the following: - - swift://user:pass@authurl.com/container/obj-id - swift://account:user:pass@authurl.com/container/obj-id - swift+http://user:pass@authurl.com/container/obj-id - swift+https://user:pass@authurl.com/container/obj-id - - When using multi-tenant a URI might look like this (a storage URL): - - swift+https://example.com/container/obj-id - - The swift+http:// URIs indicate there is an HTTP authentication URL. - The default for Swift is an HTTPS authentication URL, so swift:// and - swift+https:// are the same... - """ - - def process_specs(self): - self.scheme = self.specs.get('scheme', 'swift+https') - self.user = self.specs.get('user') - self.key = self.specs.get('key') - self.auth_or_store_url = self.specs.get('auth_or_store_url') - self.container = self.specs.get('container') - self.obj = self.specs.get('obj') - - def _get_credstring(self): - if self.user and self.key: - return '%s:%s' % (urllib.quote(self.user), urllib.quote(self.key)) - return '' - - def get_uri(self, credentials_included=True): - auth_or_store_url = self.auth_or_store_url - if auth_or_store_url.startswith('http://'): - auth_or_store_url = auth_or_store_url[len('http://'):] - elif auth_or_store_url.startswith('https://'): - auth_or_store_url = auth_or_store_url[len('https://'):] - - credstring = self._get_credstring() - auth_or_store_url = auth_or_store_url.strip('/') - container = self.container.strip('/') - obj = self.obj.strip('/') - - if not credentials_included: - #Used only in case of an add - #Get the current store from config - store = CONF.default_swift_reference - - return '%s://%s/%s/%s' % ('swift+config', store, container, obj) - if self.scheme == 'swift+config': - if self.ssl_enabled == True: - self.scheme = 'swift+https' - else: - self.scheme = 'swift+http' - if credstring != '': - credstring = "%s@" % credstring - return '%s://%s%s/%s/%s' % (self.scheme, credstring, auth_or_store_url, - container, obj) - - def _get_conf_value_from_account_ref(self, netloc): - try: - self.user = SWIFT_STORE_REF_PARAMS[netloc]['user'] - self.key = SWIFT_STORE_REF_PARAMS[netloc]['key'] - netloc = SWIFT_STORE_REF_PARAMS[netloc]['auth_address'] - self.ssl_enabled = True - if netloc != '': - if netloc.startswith('http://'): - self.ssl_enabled = False - netloc = netloc[len('http://'):] - elif netloc.startswith('https://'): - netloc = netloc[len('https://'):] - except KeyError: - reason = _("Badly formed Swift URI. Credentials not found for " - "account reference") - LOG.info(reason) - raise exception.BadStoreUri(message=reason) - return netloc - - def _form_uri_parts(self, netloc, path): - if netloc != '': - # > Python 2.6.1 - if '@' in netloc: - creds, netloc = netloc.split('@') - else: - creds = None - else: - # Python 2.6.1 compat - # see lp659445 and Python issue7904 - if '@' in path: - creds, path = path.split('@') - else: - creds = None - netloc = path[0:path.find('/')].strip('/') - path = path[path.find('/'):].strip('/') - if creds: - cred_parts = creds.split(':') - if len(cred_parts) < 2: - reason = _("Badly formed credentials in Swift URI.") - LOG.info(reason) - raise exception.BadStoreUri(message=reason) - key = cred_parts.pop() - user = ':'.join(cred_parts) - creds = urllib.unquote(creds) - try: - self.user, self.key = creds.rsplit(':', 1) - except exception.BadStoreConfiguration: - self.user = urllib.unquote(user) - self.key = urllib.unquote(key) - else: - self.user = None - self.key = None - return netloc, path - - def _form_auth_or_store_url(self, netloc, path): - path_parts = path.split('/') - try: - self.obj = path_parts.pop() - self.container = path_parts.pop() - if not netloc.startswith('http'): - # push hostname back into the remaining to build full authurl - path_parts.insert(0, netloc) - self.auth_or_store_url = '/'.join(path_parts) - except IndexError: - reason = _("Badly formed Swift URI.") - LOG.info(reason) - raise exception.BadStoreUri(message=reason) - - def parse_uri(self, uri): - """ - Parse URLs. This method fixes an issue where credentials specified - in the URL are interpreted differently in Python 2.6.1+ than prior - versions of Python. It also deals with the peculiarity that new-style - Swift URIs have where a username can contain a ':', like so: - - swift://account:user:pass@authurl.com/container/obj - and for system created locations with account reference - swift+config://account_reference/container/obj - """ - # Make sure that URIs that contain multiple schemes, such as: - # swift://user:pass@http://authurl.com/v1/container/obj - # are immediately rejected. - if uri.count('://') != 1: - reason = _("URI cannot contain more than one occurrence " - "of a scheme. If you have specified a URI like " - "swift://user:pass@http://authurl.com/v1/container/obj" - ", you need to change it to use the " - "swift+http:// scheme, like so: " - "swift+http://user:pass@authurl.com/v1/container/obj") - LOG.info(_LI("Invalid store URI: %(reason)s"), {'reason': reason}) - raise exception.BadStoreUri(message=reason) - - pieces = urlparse.urlparse(uri) - assert pieces.scheme in ('swift', 'swift+http', 'swift+https', - 'swift+config') - - self.scheme = pieces.scheme - netloc = pieces.netloc - path = pieces.path.lstrip('/') - - # NOTE(Sridevi): Fix to map the account reference to the - # corresponding CONF value - if self.scheme == 'swift+config': - netloc = self._get_conf_value_from_account_ref(netloc) - else: - netloc, path = self._form_uri_parts(netloc, path) - - self._form_auth_or_store_url(netloc, path) - - @property - def swift_url(self): - """ - Creates a fully-qualified auth address that the Swift client library - can use. The scheme for the auth_address is determined using the scheme - included in the `location` field. - - HTTPS is assumed, unless 'swift+http' is specified. - """ - if self.auth_or_store_url.startswith('http'): - return self.auth_or_store_url - else: - if self.scheme == 'swift+config': - if self.ssl_enabled == True: - self.scheme = 'swift+https' - else: - self.scheme = 'swift+http' - if self.scheme in ('swift+https', 'swift'): - auth_scheme = 'https://' - else: - auth_scheme = 'http://' - - return ''.join([auth_scheme, self.auth_or_store_url]) - - -def Store(context=None, loc=None, configure=True): - if (CONF.swift_store_multi_tenant and - (loc is None or loc.store_location.user is None)): - return MultiTenantStore(context, loc, configure=configure) - return SingleTenantStore(context, loc, configure=configure) - - -class BaseStore(glance.store.base.Store): - READ_CHUNKSIZE = 64 * units.Ki - - def get_schemes(self): - return ('swift+https', 'swift', 'swift+http', 'swift+config') - - def configure(self): - _obj_size = self._option_get('swift_store_large_object_size') - self.large_object_size = _obj_size * ONE_MB - _chunk_size = self._option_get('swift_store_large_object_chunk_size') - self.large_object_chunk_size = _chunk_size * ONE_MB - self.admin_tenants = CONF.swift_store_admin_tenants - self.region = CONF.swift_store_region - self.service_type = CONF.swift_store_service_type - self.endpoint_type = CONF.swift_store_endpoint_type - self.snet = CONF.swift_enable_snet - self.insecure = CONF.swift_store_auth_insecure - self.ssl_compression = CONF.swift_store_ssl_compression - - def _get_object(self, location, connection=None, start=None): - if not connection: - connection = self.get_connection(location) - headers = {} - if start is not None: - bytes_range = 'bytes=%d-' % start - headers = {'Range': bytes_range} - - try: - resp_headers, resp_body = connection.get_object( - container=location.container, obj=location.obj, - resp_chunk_size=self.READ_CHUNKSIZE, headers=headers) - except swiftclient.ClientException as e: - if e.http_status == httplib.NOT_FOUND: - msg = _("Swift could not find object %s.") % location.obj - LOG.warn(msg) - raise exception.NotFound(msg) - else: - raise - - return (resp_headers, resp_body) - - def get(self, location, connection=None): - location = location.store_location - (resp_headers, resp_body) = self._get_object(location, connection) - - class ResponseIndexable(glance.store.Indexable): - def another(self): - try: - return self.wrapped.next() - except StopIteration: - return '' - - length = int(resp_headers.get('content-length', 0)) - if CONF.swift_store_retry_get_count > 0: - resp_body = swift_retry_iter(resp_body, length, self, location) - return (ResponseIndexable(resp_body, length), length) - - def get_size(self, location, connection=None): - location = location.store_location - if not connection: - connection = self.get_connection(location) - try: - resp_headers = connection.head_object( - container=location.container, obj=location.obj) - return int(resp_headers.get('content-length', 0)) - except Exception: - return 0 - - def _option_get(self, param): - result = getattr(CONF, param) - if not result: - reason = (_("Could not find %(param)s in configuration options.") - % param) - LOG.error(reason) - raise exception.BadStoreConfiguration(store_name="swift", - reason=reason) - return result - - def _delete_stale_chunks(self, connection, container, chunk_list): - for chunk in chunk_list: - LOG.debug("Deleting chunk %s" % chunk) - try: - connection.delete_object(container, chunk) - except Exception: - msg = _("Failed to delete orphaned chunk " - "%(container)s/%(chunk)s") - LOG.exception(msg % {'container': container, - 'chunk': chunk}) - - def add(self, image_id, image_file, image_size, connection=None): - location = self.create_location(image_id) - if not connection: - connection = self.get_connection(location) - - self._create_container_if_missing(location.container, connection) - - LOG.debug("Adding image object '%(obj_name)s' " - "to Swift" % dict(obj_name=location.obj)) - try: - if image_size > 0 and image_size < self.large_object_size: - # Image size is known, and is less than large_object_size. - # Send to Swift with regular PUT. - obj_etag = connection.put_object(location.container, - location.obj, image_file, - content_length=image_size) - else: - # Write the image into Swift in chunks. - chunk_id = 1 - if image_size > 0: - total_chunks = str(int( - math.ceil(float(image_size) / - float(self.large_object_chunk_size)))) - else: - # image_size == 0 is when we don't know the size - # of the image. This can occur with older clients - # that don't inspect the payload size. - LOG.debug("Cannot determine image size. Adding as a " - "segmented object to Swift.") - total_chunks = '?' - - checksum = hashlib.md5() - written_chunks = [] - combined_chunks_size = 0 - while True: - chunk_size = self.large_object_chunk_size - if image_size == 0: - content_length = None - else: - left = image_size - combined_chunks_size - if left == 0: - break - if chunk_size > left: - chunk_size = left - content_length = chunk_size - - chunk_name = "%s-%05d" % (location.obj, chunk_id) - reader = ChunkReader(image_file, checksum, chunk_size) - try: - chunk_etag = connection.put_object( - location.container, chunk_name, reader, - content_length=content_length) - written_chunks.append(chunk_name) - except Exception: - # Delete orphaned segments from swift backend - with excutils.save_and_reraise_exception(): - LOG.exception(_("Error during chunked upload to " - "backend, deleting stale chunks")) - self._delete_stale_chunks(connection, - location.container, - written_chunks) - - bytes_read = reader.bytes_read - msg = ("Wrote chunk %(chunk_name)s (%(chunk_id)d/" - "%(total_chunks)s) of length %(bytes_read)d " - "to Swift returning MD5 of content: " - "%(chunk_etag)s" % - {'chunk_name': chunk_name, - 'chunk_id': chunk_id, - 'total_chunks': total_chunks, - 'bytes_read': bytes_read, - 'chunk_etag': chunk_etag}) - LOG.debug(msg) - - if bytes_read == 0: - # Delete the last chunk, because it's of zero size. - # This will happen if size == 0. - LOG.debug("Deleting final zero-length chunk") - connection.delete_object(location.container, - chunk_name) - break - - chunk_id += 1 - combined_chunks_size += bytes_read - - # In the case we have been given an unknown image size, - # set the size to the total size of the combined chunks. - if image_size == 0: - image_size = combined_chunks_size - - # Now we write the object manifest and return the - # manifest's etag... - manifest = "%s/%s-" % (location.container, location.obj) - headers = {'ETag': hashlib.md5("").hexdigest(), - 'X-Object-Manifest': manifest} - - # The ETag returned for the manifest is actually the - # MD5 hash of the concatenated checksums of the strings - # of each chunk...so we ignore this result in favour of - # the MD5 of the entire image file contents, so that - # users can verify the image file contents accordingly - connection.put_object(location.container, location.obj, - None, headers=headers) - obj_etag = checksum.hexdigest() - - # NOTE: We return the user and key here! Have to because - # location is used by the API server to return the actual - # image data. We *really* should consider NOT returning - # the location attribute from GET /images/ and - # GET /images/details - if swift_store_utils.is_multiple_swift_store_accounts_enabled(): - include_creds = False - else: - include_creds = True - - return (location.get_uri(credentials_included=include_creds), - image_size, obj_etag, {}) - except swiftclient.ClientException as e: - if e.http_status == httplib.CONFLICT: - raise exception.Duplicate(_("Swift already has an image at " - "this location")) - msg = (_("Failed to add object to Swift.\n" - "Got error from Swift: %s") % utils.exception_to_str(e)) - LOG.error(msg) - raise glance.store.BackendException(msg) - - def delete(self, location, connection=None): - location = location.store_location - if not connection: - connection = self.get_connection(location) - - try: - # We request the manifest for the object. If one exists, - # that means the object was uploaded in chunks/segments, - # and we need to delete all the chunks as well as the - # manifest. - manifest = None - try: - headers = connection.head_object( - location.container, location.obj) - manifest = headers.get('x-object-manifest') - except swiftclient.ClientException as e: - if e.http_status != httplib.NOT_FOUND: - raise - if manifest: - # Delete all the chunks before the object manifest itself - obj_container, obj_prefix = manifest.split('/', 1) - segments = connection.get_container( - obj_container, prefix=obj_prefix)[1] - for segment in segments: - # TODO(jaypipes): This would be an easy area to parallelize - # since we're simply sending off parallelizable requests - # to Swift to delete stuff. It's not like we're going to - # be hogging up network or file I/O here... - connection.delete_object(obj_container, - segment['name']) - - # Delete object (or, in segmented case, the manifest) - connection.delete_object(location.container, location.obj) - - except swiftclient.ClientException as e: - if e.http_status == httplib.NOT_FOUND: - msg = _("Swift could not find image at URI.") - raise exception.NotFound(msg) - else: - raise - - def _create_container_if_missing(self, container, connection): - """ - Creates a missing container in Swift if the - ``swift_store_create_container_on_put`` option is set. - - :param container: Name of container to create - :param connection: Connection to swift service - """ - try: - connection.head_container(container) - except swiftclient.ClientException as e: - if e.http_status == httplib.NOT_FOUND: - if CONF.swift_store_create_container_on_put: - try: - msg = (_LI("Creating swift container %(container)s") % - {'container': container}) - LOG.info(msg) - connection.put_container(container) - except swiftclient.ClientException as e: - msg = (_("Failed to add container to Swift.\n" - "Got error from Swift: %(e)s") % {'e': e}) - raise glance.store.BackendException(msg) - else: - msg = (_("The container %(container)s does not exist in " - "Swift. Please set the " - "swift_store_create_container_on_put option" - "to add container to Swift automatically.") % - {'container': container}) - raise glance.store.BackendException(msg) - else: - raise - - def get_connection(self): - raise NotImplementedError() - - def create_location(self): - raise NotImplementedError() - - -class SingleTenantStore(BaseStore): - EXAMPLE_URL = "swift://:@//" - - def configure(self): - super(SingleTenantStore, self).configure() - self.auth_version = self._option_get('swift_store_auth_version') - - def configure_add(self): - default_swift_reference = \ - SWIFT_STORE_REF_PARAMS.get( - CONF.default_swift_reference) - if default_swift_reference: - self.auth_address = default_swift_reference.get('auth_address') - if (not default_swift_reference) or (not self.auth_address): - reason = _("A value for swift_store_auth_address is required.") - LOG.error(reason) - raise exception.BadStoreConfiguration(store_name="swift", - reason=reason) - if self.auth_address.startswith('http://'): - self.scheme = 'swift+http' - else: - self.scheme = 'swift+https' - self.container = CONF.swift_store_container - self.user = default_swift_reference.get('user') - self.key = default_swift_reference.get('key') - - if not (self.user or self.key): - reason = _("A value for swift_store_ref_params is required.") - LOG.error(reason) - raise exception.BadStoreConfiguration(store_name="swift", - reason=reason) - - def create_location(self, image_id): - specs = {'scheme': self.scheme, - 'container': self.container, - 'obj': str(image_id), - 'auth_or_store_url': self.auth_address, - 'user': self.user, - 'key': self.key} - return StoreLocation(specs) - - def validate_location(self, uri): - pieces = urlparse.urlparse(uri) - if pieces.scheme in ['swift+config']: - reason = (_("Location credentials are invalid")) - raise exception.BadStoreUri(message=reason) - - def get_connection(self, location): - if not location.user: - reason = _("Location is missing user:password information.") - LOG.info(reason) - raise exception.BadStoreUri(message=reason) - - auth_url = location.swift_url - if not auth_url.endswith('/'): - auth_url += '/' - - if self.auth_version == '2': - try: - tenant_name, user = location.user.split(':') - except ValueError: - reason = (_("Badly formed tenant:user '%(user)s' in " - "Swift URI") % {'user': location.user}) - LOG.info(reason) - raise exception.BadStoreUri(message=reason) - else: - tenant_name = None - user = location.user - - os_options = {} - if self.region: - os_options['region_name'] = self.region - os_options['endpoint_type'] = self.endpoint_type - os_options['service_type'] = self.service_type - - return swiftclient.Connection( - auth_url, user, location.key, insecure=self.insecure, - tenant_name=tenant_name, snet=self.snet, - auth_version=self.auth_version, os_options=os_options, - ssl_compression=self.ssl_compression) - - -class MultiTenantStore(BaseStore): - EXAMPLE_URL = "swift:////" - - def configure_add(self): - self.container = CONF.swift_store_container - if self.context is None: - reason = _("Multi-tenant Swift storage requires a context.") - raise exception.BadStoreConfiguration(store_name="swift", - reason=reason) - if self.context.service_catalog is None: - reason = _("Multi-tenant Swift storage requires " - "a service catalog.") - raise exception.BadStoreConfiguration(store_name="swift", - reason=reason) - self.storage_url = auth.get_endpoint( - self.context.service_catalog, service_type=self.service_type, - endpoint_region=self.region, endpoint_type=self.endpoint_type) - if self.storage_url.startswith('http://'): - self.scheme = 'swift+http' - else: - self.scheme = 'swift+https' - - def delete(self, location, connection=None): - if not connection: - connection = self.get_connection(location.store_location) - super(MultiTenantStore, self).delete(location, connection) - connection.delete_container(location.store_location.container) - - def set_acls(self, location, public=False, read_tenants=None, - write_tenants=None, connection=None): - location = location.store_location - if not connection: - connection = self.get_connection(location) - - if read_tenants is None: - read_tenants = [] - if write_tenants is None: - write_tenants = [] - - headers = {} - if public: - headers['X-Container-Read'] = ".r:*,.rlistings" - elif read_tenants: - headers['X-Container-Read'] = ','.join('%s:*' % i - for i in read_tenants) - else: - headers['X-Container-Read'] = '' - - write_tenants.extend(self.admin_tenants) - if write_tenants: - headers['X-Container-Write'] = ','.join('%s:*' % i - for i in write_tenants) - else: - headers['X-Container-Write'] = '' - - try: - connection.post_container(location.container, headers=headers) - except swiftclient.ClientException as e: - if e.http_status == httplib.NOT_FOUND: - msg = _("Swift could not find image at URI.") - raise exception.NotFound(msg) - else: - raise - - def create_location(self, image_id): - specs = {'scheme': self.scheme, - 'container': self.container + '_' + str(image_id), - 'obj': str(image_id), - 'auth_or_store_url': self.storage_url} - return StoreLocation(specs) - - def get_connection(self, location): - return swiftclient.Connection( - None, self.context.user, None, - preauthurl=location.swift_url, - preauthtoken=self.context.auth_tok, - tenant_name=self.context.tenant, - auth_version='2', snet=self.snet, insecure=self.insecure, - ssl_compression=self.ssl_compression) - - -class ChunkReader(object): - def __init__(self, fd, checksum, total): - self.fd = fd - self.checksum = checksum - self.total = total - self.bytes_read = 0 - - def read(self, i): - left = self.total - self.bytes_read - if i > left: - i = left - result = self.fd.read(i) - self.bytes_read += len(result) - self.checksum.update(result) - return result diff --git a/glance/store/vmware_datastore.py b/glance/store/vmware_datastore.py deleted file mode 100644 index eb86f9ea62..0000000000 --- a/glance/store/vmware_datastore.py +++ /dev/null @@ -1,503 +0,0 @@ -# Copyright 2014 OpenStack, LLC -# Copyright (c) 2014 VMware, Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Storage backend for VMware Datastore""" - -import hashlib -import httplib -import os - -import netaddr -from oslo.config import cfg -from oslo.vmware import api -from retrying import retry -import six.moves.urllib.parse as urlparse - -from glance.common import exception -from glance.openstack.common import excutils -from glance.openstack.common import gettextutils -import glance.openstack.common.log as logging -from glance.openstack.common import units -import glance.store -import glance.store.base -import glance.store.location - - -LOG = logging.getLogger(__name__) -_LE = gettextutils._LE -_LI = gettextutils._LI - -MAX_REDIRECTS = 5 -DEFAULT_STORE_IMAGE_DIR = '/openstack_glance' -DEFAULT_ESX_DATACENTER_PATH = 'ha-datacenter' -DS_URL_PREFIX = '/folder' -STORE_SCHEME = 'vsphere' - -# check that datacenter/datastore combination is valid -_datastore_info_valid = False - -vmware_opts = [ - cfg.StrOpt('vmware_server_host', - help=_('ESX/ESXi or vCenter Server target system. ' - 'The server value can be an IP address or a DNS name.')), - cfg.StrOpt('vmware_server_username', - help=_('Username for authenticating with ' - 'VMware ESX/VC server.')), - cfg.StrOpt('vmware_server_password', - help=_('Password for authenticating with ' - 'VMware ESX/VC server.'), - secret=True), - cfg.StrOpt('vmware_datacenter_path', - default=DEFAULT_ESX_DATACENTER_PATH, - help=_('Inventory path to a datacenter. ' - 'If the vmware_server_host specified is an ESX/ESXi, ' - 'the vmware_datacenter_path is optional. If specified, ' - 'it should be "ha-datacenter".')), - cfg.StrOpt('vmware_datastore_name', - help=_('Datastore associated with the datacenter.')), - cfg.IntOpt('vmware_api_retry_count', - default=10, - help=_('Number of times VMware ESX/VC server API must be ' - 'retried upon connection related issues.')), - cfg.IntOpt('vmware_task_poll_interval', - default=5, - help=_('The interval used for polling remote tasks ' - 'invoked on VMware ESX/VC server.')), - cfg.StrOpt('vmware_store_image_dir', - default=DEFAULT_STORE_IMAGE_DIR, - help=_('The name of the directory where the glance images ' - 'will be stored in the VMware datastore.')), - cfg.BoolOpt('vmware_api_insecure', - default=False, - help=_('Allow to perform insecure SSL requests to ESX/VC.')), -] - -CONF = cfg.CONF -CONF.register_opts(vmware_opts) - - -def is_valid_ipv6(address): - try: - return netaddr.valid_ipv6(address) - except Exception: - return False - - -def http_response_iterator(conn, response, size): - """Return an iterator for a file-like object. - - :param conn: HTTP(S) Connection - :param response: httplib.HTTPResponse object - :param size: Chunk size to iterate with - """ - try: - chunk = response.read(size) - while chunk: - yield chunk - chunk = response.read(size) - finally: - conn.close() - - -class _Reader(object): - - def __init__(self, data): - self._size = 0 - self.data = data - self.checksum = hashlib.md5() - - def read(self, size=None): - result = self.data.read(size) - self._size += len(result) - self.checksum.update(result) - return result - - def rewind(self): - try: - self.data.seek(0) - self._size = 0 - self.checksum = hashlib.md5() - except IOError: - with excutils.save_and_reraise_exception(): - LOG.exception(_LE('Failed to rewind image content')) - - @property - def size(self): - return self._size - - -class _ChunkReader(_Reader): - - def __init__(self, data, blocksize=8192): - self.blocksize = blocksize - self.current_chunk = "" - self.closed = False - super(_ChunkReader, self).__init__(data) - - def read(self, size=None): - ret = "" - while size is None or size >= len(self.current_chunk): - ret += self.current_chunk - if size is not None: - size -= len(self.current_chunk) - if self.closed: - self.current_chunk = "" - break - self._get_chunk() - else: - ret += self.current_chunk[:size] - self.current_chunk = self.current_chunk[size:] - return ret - - def _get_chunk(self): - if not self.closed: - chunk = self.data.read(self.blocksize) - chunk_len = len(chunk) - self._size += chunk_len - self.checksum.update(chunk) - if chunk: - self.current_chunk = '%x\r\n%s\r\n' % (chunk_len, chunk) - else: - self.current_chunk = '0\r\n\r\n' - self.closed = True - - -class StoreLocation(glance.store.location.StoreLocation): - """Class describing an VMware URI. - - An VMware URI can look like any of the following: - vsphere://server_host/folder/file_path?dcPath=dc_path&dsName=ds_name - """ - - def process_specs(self): - self.scheme = self.specs.get('scheme', STORE_SCHEME) - self.server_host = self.specs.get('server_host') - self.path = os.path.join(DS_URL_PREFIX, - self.specs.get('image_dir').strip('/'), - self.specs.get('image_id')) - dc_path = self.specs.get('datacenter_path') - if dc_path is not None: - param_list = {'dcPath': self.specs.get('datacenter_path'), - 'dsName': self.specs.get('datastore_name')} - else: - param_list = {'dsName': self.specs.get('datastore_name')} - self.query = urlparse.urlencode(param_list) - - def get_uri(self): - if is_valid_ipv6(self.server_host): - base_url = '%s://[%s]%s' % (self.scheme, - self.server_host, self.path) - else: - base_url = '%s://%s%s' % (self.scheme, - self.server_host, self.path) - - return '%s?%s' % (base_url, self.query) - - def _is_valid_path(self, path): - return path.startswith( - os.path.join(DS_URL_PREFIX, - CONF.vmware_store_image_dir.strip('/'))) - - def parse_uri(self, uri): - if not uri.startswith('%s://' % STORE_SCHEME): - reason = (_("URI must start with %s://") % STORE_SCHEME) - LOG.info(reason) - raise exception.BadStoreUri(message=reason) - (self.scheme, self.server_host, - path, params, query, fragment) = urlparse.urlparse(uri) - if not query: - path = path.split('?') - if self._is_valid_path(path[0]): - self.path = path[0] - self.query = path[1] - return - elif self._is_valid_path(path): - self.path = path - self.query = query - return - reason = _('Badly formed VMware datastore URI') - LOG.info(reason) - raise exception.BadStoreUri(message=reason) - - -class Store(glance.store.base.Store): - """An implementation of the VMware datastore adapter.""" - - WRITE_CHUNKSIZE = units.Mi - - def get_schemes(self): - return (STORE_SCHEME,) - - def _sanity_check(self): - if CONF.vmware_api_retry_count <= 0: - msg = _("vmware_api_retry_count should be greater than zero") - LOG.error(msg) - raise exception.BadStoreConfiguration( - store_name='vmware_datastore', reason=msg) - if CONF.vmware_task_poll_interval <= 0: - msg = _("vmware_task_poll_interval should be greater than zero") - LOG.error(msg) - raise exception.BadStoreConfiguration( - store_name='vmware_datastore', reason=msg) - - def configure(self): - self._sanity_check() - self.scheme = STORE_SCHEME - self.server_host = self._option_get('vmware_server_host') - self.server_username = self._option_get('vmware_server_username') - self.server_password = self._option_get('vmware_server_password') - self.api_retry_count = CONF.vmware_api_retry_count - self.task_poll_interval = CONF.vmware_task_poll_interval - self.api_insecure = CONF.vmware_api_insecure - self._create_session() - - def configure_add(self): - self.datacenter_path = CONF.vmware_datacenter_path - self.datastore_name = self._option_get('vmware_datastore_name') - global _datastore_info_valid - if not _datastore_info_valid: - search_index_moref = self._service_content.searchIndex - - inventory_path = ('%s/datastore/%s' - % (self.datacenter_path, self.datastore_name)) - ds_moref = self._session.invoke_api(self._session.vim, - 'FindByInventoryPath', - search_index_moref, - inventoryPath=inventory_path) - if ds_moref is None: - msg = (_("Could not find datastore %(ds_name)s " - "in datacenter %(dc_path)s") - % {'ds_name': self.datastore_name, - 'dc_path': self.datacenter_path}) - LOG.error(msg) - raise exception.BadStoreConfiguration( - store_name='vmware_datastore', reason=msg) - else: - _datastore_info_valid = True - self.store_image_dir = CONF.vmware_store_image_dir - - def _create_session(self): - self._session = api.VMwareAPISession( - self.server_host, self.server_username, self.server_password, - self.api_retry_count, self.task_poll_interval) - self._service_content = self._session.vim.service_content - - def _option_get(self, param): - result = getattr(CONF, param) - if not result: - reason = (_("Could not find %(param)s in configuration " - "options.") % {'param': param}) - raise exception.BadStoreConfiguration( - store_name='vmware_datastore', reason=reason) - return result - - def _build_vim_cookie_header(self, vim_cookies): - """Build ESX host session cookie header.""" - if len(list(vim_cookies)) > 0: - cookie = list(vim_cookies)[0] - return cookie.name + '=' + cookie.value - - def _session_not_authenticated(exc): - if isinstance(exc, exception.NotAuthenticated): - LOG.info(_LI("Store session is not authenticated, retry attempt")) - return True - return False - - @retry(stop_max_attempt_number=CONF.vmware_api_retry_count + 1, - retry_on_exception=_session_not_authenticated) - def add(self, image_id, image_file, image_size): - """Stores an image file with supplied identifier to the backend - storage system and returns a tuple containing information - about the stored image. - - :param image_id: The opaque image identifier - :param image_file: The image data to write, as a file-like object - :param image_size: The size of the image data to write, in bytes - :retval tuple of URL in backing store, bytes written, checksum - and a dictionary with storage system specific information - :raises `glance.common.exception.Duplicate` if the image already - existed - `glance.common.exception.UnexpectedStatus` if the upload - request returned an unexpected status. The expected responses - are 201 Created and 200 OK. - """ - if image_size > 0: - headers = {'Content-Length': image_size} - image_file = _Reader(image_file) - else: - # NOTE (arnaud): use chunk encoding when the image is still being - # generated by the server (ex: stream optimized disks generated by - # Nova). - headers = {'Transfer-Encoding': 'chunked'} - image_file = _ChunkReader(image_file) - loc = StoreLocation({'scheme': self.scheme, - 'server_host': self.server_host, - 'image_dir': self.store_image_dir, - 'datacenter_path': self.datacenter_path, - 'datastore_name': self.datastore_name, - 'image_id': image_id}) - cookie = self._build_vim_cookie_header( - self._session.vim.client.options.transport.cookiejar) - headers = dict(headers.items() + {'Cookie': cookie}.items()) - try: - conn = self._get_http_conn('PUT', loc, headers, - content=image_file) - res = conn.getresponse() - except Exception: - with excutils.save_and_reraise_exception(): - LOG.exception(_LE('Failed to upload content of image ' - '%(image)s'), {'image': image_id}) - - if res.status == httplib.UNAUTHORIZED: - self._create_session() - image_file.rewind() - raise exception.NotAuthenticated() - - if res.status == httplib.CONFLICT: - raise exception.Duplicate(_("Image file %(image_id)s already " - "exists!") % {'image_id': image_id}) - - if res.status not in (httplib.CREATED, httplib.OK): - msg = (_LE('Failed to upload content of image %(image)s') % - {'image': image_id}) - LOG.error(msg) - raise exception.UnexpectedStatus(status=res.status, - body=res.read()) - - return (loc.get_uri(), image_file.size, - image_file.checksum.hexdigest(), {}) - - def get(self, location): - """Takes a `glance.store.location.Location` object that indicates - where to find the image file, and returns a tuple of generator - (for reading the image file) and image_size - - :param location: `glance.store.location.Location` object, supplied - from glance.store.location.get_location_from_uri() - """ - conn, resp, content_length = self._query(location, 'GET') - iterator = http_response_iterator(conn, resp, self.READ_CHUNKSIZE) - - class ResponseIndexable(glance.store.Indexable): - - def another(self): - try: - return self.wrapped.next() - except StopIteration: - return '' - - return (ResponseIndexable(iterator, content_length), content_length) - - def get_size(self, location): - """Takes a `glance.store.location.Location` object that indicates - where to find the image file, and returns the size - - :param location: `glance.store.location.Location` object, supplied - from glance.store.location.get_location_from_uri() - """ - return self._query(location, 'HEAD')[2] - - def delete(self, location): - """Takes a `glance.store.location.Location` object that indicates - where to find the image file to delete - - :location `glance.store.location.Location` object, supplied - from glance.store.location.get_location_from_uri() - :raises NotFound if image does not exist - """ - file_path = '[%s] %s' % ( - self.datastore_name, - location.store_location.path[len(DS_URL_PREFIX):]) - search_index_moref = self._service_content.searchIndex - dc_moref = self._session.invoke_api(self._session.vim, - 'FindByInventoryPath', - search_index_moref, - inventoryPath=self.datacenter_path) - delete_task = self._session.invoke_api( - self._session.vim, - 'DeleteDatastoreFile_Task', - self._service_content.fileManager, - name=file_path, - datacenter=dc_moref) - try: - self._session.wait_for_task(delete_task) - except Exception: - with excutils.save_and_reraise_exception(): - LOG.exception(_LE('Failed to delete image %(image)s content.'), - {'image': location.image_id}) - - @retry(stop_max_attempt_number=CONF.vmware_api_retry_count + 1, - retry_on_exception=_session_not_authenticated) - def _query(self, location, method, depth=0): - if depth > MAX_REDIRECTS: - msg = ("The HTTP URL exceeded %(max_redirects)s maximum " - "redirects.", {'max_redirects': MAX_REDIRECTS}) - LOG.debug(msg) - raise exception.MaxRedirectsExceeded(redirects=MAX_REDIRECTS) - loc = location.store_location - cookie = self._build_vim_cookie_header( - self._session.vim.client.options.transport.cookiejar) - headers = {'Cookie': cookie} - try: - conn = self._get_http_conn(method, loc, headers) - resp = conn.getresponse() - except Exception: - with excutils.save_and_reraise_exception(): - LOG.exception(_LE('Failed to access image %(image)s content.'), - {'image': location.image_id}) - if resp.status >= 400: - if resp.status == httplib.UNAUTHORIZED: - self._create_session() - raise exception.NotAuthenticated() - if resp.status == httplib.NOT_FOUND: - msg = 'VMware datastore could not find image at URI.' - LOG.debug(msg) - raise exception.NotFound(msg) - reason = (_('HTTP request returned a %(status)s status code.') - % {'status': resp.status}) - LOG.info(reason) - raise exception.BadStoreUri(message=reason) - location_header = resp.getheader('location') - if location_header: - if resp.status not in (301, 302): - reason = (_("The HTTP URL %(path)s attempted to redirect " - "with an invalid %(status)s status code.") - % {'path': loc.path, 'status': resp.status}) - LOG.info(reason) - raise exception.BadStoreUri(message=reason) - location_class = glance.store.location.Location - new_loc = location_class(location.store_name, - location.store_location.__class__, - uri=location_header, - image_id=location.image_id, - store_specs=location.store_specs) - return self._query(new_loc, method, depth + 1) - content_length = int(resp.getheader('content-length', 0)) - - return (conn, resp, content_length) - - def _get_http_conn(self, method, loc, headers, content=None): - conn_class = self._get_http_conn_class() - conn = conn_class(loc.server_host) - url = urlparse.quote('%s?%s' % (loc.path, loc.query)) - conn.request(method, url, content, headers) - - return conn - - def _get_http_conn_class(self): - if self.api_insecure: - return httplib.HTTPConnection - return httplib.HTTPSConnection diff --git a/glance/tests/functional/__init__.py b/glance/tests/functional/__init__.py index 872df6299b..297f4aa603 100644 --- a/glance/tests/functional/__init__.py +++ b/glance/tests/functional/__init__.py @@ -39,7 +39,6 @@ import testtools from glance.common import utils from glance.db.sqlalchemy import api as db_api from glance.openstack.common import jsonutils -from glance.openstack.common import units from glance import tests as glance_tests from glance.tests import utils as test_utils @@ -280,28 +279,6 @@ class ApiServer(Server): self.scrubber_datadir = os.path.join(self.test_dir, "scrubber") self.log_file = os.path.join(self.test_dir, "api.log") self.image_size_cap = 1099511627776 - self.s3_store_host = "s3.amazonaws.com" - self.s3_store_access_key = "" - self.s3_store_secret_key = "" - self.s3_store_bucket = "" - self.s3_store_bucket_url_format = "" - self.swift_store_auth_version = kwargs.get("swift_store_auth_version", - "2") - self.swift_store_auth_address = kwargs.get("swift_store_auth_address", - "") - self.swift_store_user = kwargs.get("swift_store_user", "") - self.swift_store_key = kwargs.get("swift_store_key", "") - self.swift_store_container = kwargs.get("swift_store_container", "") - self.swift_store_create_container_on_put = kwargs.get( - "swift_store_create_container_on_put", "True") - self.swift_store_large_object_size = 5 * units.Ki - self.swift_store_large_object_chunk_size = 200 - self.swift_store_multi_tenant = False - self.swift_store_admin_tenants = [] - self.rbd_store_ceph_conf = "" - self.rbd_store_pool = "" - self.rbd_store_user = "" - self.rbd_store_chunk_size = 4 self.delayed_delete = delayed_delete self.owner_is_tenant = True self.workers = 0 @@ -333,8 +310,6 @@ class ApiServer(Server): verbose = %(verbose)s debug = %(debug)s default_log_levels = eventlet.wsgi.server=DEBUG -filesystem_store_datadir=%(image_dir)s -default_store = %(default_store)s bind_host = 127.0.0.1 bind_port = %(bind_port)s key_file = %(key_file)s @@ -344,25 +319,6 @@ registry_host = 127.0.0.1 registry_port = %(registry_port)s log_file = %(log_file)s image_size_cap = %(image_size_cap)d -s3_store_host = %(s3_store_host)s -s3_store_access_key = %(s3_store_access_key)s -s3_store_secret_key = %(s3_store_secret_key)s -s3_store_bucket = %(s3_store_bucket)s -s3_store_bucket_url_format = %(s3_store_bucket_url_format)s -swift_store_auth_version = %(swift_store_auth_version)s -swift_store_auth_address = %(swift_store_auth_address)s -swift_store_user = %(swift_store_user)s -swift_store_key = %(swift_store_key)s -swift_store_container = %(swift_store_container)s -swift_store_create_container_on_put = %(swift_store_create_container_on_put)s -swift_store_large_object_size = %(swift_store_large_object_size)s -swift_store_large_object_chunk_size = %(swift_store_large_object_chunk_size)s -swift_store_multi_tenant = %(swift_store_multi_tenant)s -swift_store_admin_tenants = %(swift_store_admin_tenants)s -rbd_store_chunk_size = %(rbd_store_chunk_size)s -rbd_store_user = %(rbd_store_user)s -rbd_store_pool = %(rbd_store_pool)s -rbd_store_ceph_conf = %(rbd_store_ceph_conf)s delayed_delete = %(delayed_delete)s owner_is_tenant = %(owner_is_tenant)s workers = %(workers)s @@ -392,6 +348,9 @@ location_strategy=%(location_strategy)s flavor = %(deployment_flavor)s [store_type_location_strategy] store_type_preference = %(store_type_location_strategy_preference)s +[glance_store] +filesystem_store_datadir=%(image_dir)s +default_store = %(default_store)s """ self.paste_conf_base = """[pipeline:glance-api] pipeline = versionnegotiation gzip unauthenticated-context rootapp @@ -540,13 +499,6 @@ class ScrubberDaemon(Server): "scrubber") self.pid_file = os.path.join(self.test_dir, "scrubber.pid") self.log_file = os.path.join(self.test_dir, "scrubber.log") - self.swift_store_auth_address = kwargs.get("swift_store_auth_address", - "") - self.swift_store_user = kwargs.get("swift_store_user", "") - self.swift_store_key = kwargs.get("swift_store_key", "") - self.swift_store_container = kwargs.get("swift_store_container", "") - self.swift_store_auth_version = kwargs.get("swift_store_auth_version", - "2") self.metadata_encryption_key = "012345678901234567890123456789ab" self.lock_path = self.test_dir @@ -566,11 +518,6 @@ scrubber_datadir = %(scrubber_datadir)s registry_host = 127.0.0.1 registry_port = %(registry_port)s metadata_encryption_key = %(metadata_encryption_key)s -swift_store_auth_address = %(swift_store_auth_address)s -swift_store_user = %(swift_store_user)s -swift_store_key = %(swift_store_key)s -swift_store_container = %(swift_store_container)s -swift_store_auth_version = %(swift_store_auth_version)s lock_path = %(lock_path)s sql_connection = %(sql_connection)s sql_idle_timeout = 3600 diff --git a/glance/tests/functional/store/__init__.py b/glance/tests/functional/store/__init__.py deleted file mode 100644 index 14a73a21fc..0000000000 --- a/glance/tests/functional/store/__init__.py +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright 2012 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import uuid - -from oslo.config import cfg -import six - -#NOTE(bcwaldon): importing this to get the default_store option -import glance.api.v1.images -from glance.common import exception - -import glance.store.location - -CONF = cfg.CONF - - -class BaseTestCase(object): - """ - Basic test cases for glance image stores. - - To run these tests on a new store X, create a test case like - - class TestXStore(BaseTestCase, testtools.TestCase): - (MULTIPLE INHERITANCE REQUIRED) - - def get_store(...): - (STORE SPECIFIC) - - def stash_image(...): - (STORE SPECIFIC) - """ - - def setUp(self): - super(BaseTestCase, self).setUp() - - def tearDown(self): - CONF.reset() - super(BaseTestCase, self).tearDown() - - def config(self, **kw): - for k, v in six.iteritems(kw): - CONF.set_override(k, v, group=None) - - def get_store(self, **kwargs): - raise NotImplementedError('get_store() must be implemented') - - def stash_image(self, image_id, image_data): - """Store image data in the backend manually - - :param image_id: image identifier - :param image_data: string representing image data fixture - :return URI referencing newly-created backend object - """ - raise NotImplementedError('stash_image is not implemented') - - def test_create_store(self): - self.config(known_stores=[self.store_cls_path]) - count = glance.store.create_stores() - self.assertEqual(7, count) - - def test_lifecycle(self): - """Add, get and delete an image""" - store = self.get_store() - - image_id = str(uuid.uuid4()) - image_data = six.StringIO('XXX') - image_checksum = 'bc9189406be84ec297464a514221406d' - try: - uri, add_size, add_checksum, _ = store.add(image_id, image_data, 3) - except NotImplementedError: - msg = 'Configured store can not add images' - self.skipTest(msg) - - self.assertEqual(3, add_size) - self.assertEqual(image_checksum, add_checksum) - - store = self.get_store() - location = glance.store.location.Location( - self.store_name, - store.get_store_location_class(), - uri=uri, - image_id=image_id) - - (get_iter, get_size) = store.get(location) - self.assertEqual(3, get_size) - self.assertEqual('XXX', ''.join(get_iter)) - - image_size = store.get_size(location) - self.assertEqual(3, image_size) - - store.delete(location) - - self.assertRaises(exception.NotFound, store.get, location) - - def test_get_remote_image(self): - """Get an image that was created externally to Glance""" - image_id = str(uuid.uuid4()) - try: - image_uri = self.stash_image(image_id, 'XXX') - except NotImplementedError: - msg = 'Configured store can not stash images' - self.skipTest(msg) - - store = self.get_store() - location = glance.store.location.Location( - self.store_name, - store.get_store_location_class(), - uri=image_uri) - - (get_iter, get_size) = store.get(location) - self.assertEqual(3, get_size) - self.assertEqual('XXX', ''.join(get_iter)) - - image_size = store.get_size(location) - self.assertEqual(3, image_size) diff --git a/glance/tests/functional/store/test_cinder.py b/glance/tests/functional/store/test_cinder.py deleted file mode 100644 index 7a53ccc6f9..0000000000 --- a/glance/tests/functional/store/test_cinder.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright 2013 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Functional tests for the Cinder store interface -""" - -import os - -import oslo.config.cfg -import testtools - -import glance.store.cinder as cinder -import glance.tests.functional.store as store_tests -import glance.tests.functional.store.test_swift as store_tests_swift -import glance.tests.utils - - -def parse_config(config): - out = {} - options = [ - 'test_cinder_store_auth_address', - 'test_cinder_store_auth_version', - 'test_cinder_store_tenant', - 'test_cinder_store_user', - 'test_cinder_store_key', - ] - - for option in options: - out[option] = config.defaults()[option] - - return out - - -class TestCinderStore(store_tests.BaseTestCase, testtools.TestCase): - - store_cls_path = 'glance.store.cinder.Store' - store_cls = glance.store.cinder.Store - store_name = 'cinder' - - def setUp(self): - config_path = os.environ.get('GLANCE_TEST_CINDER_CONF') - if not config_path: - msg = "GLANCE_TEST_CINDER_CONF environ not set." - self.skipTest(msg) - - oslo.config.cfg.CONF(args=[], default_config_files=[config_path]) - raw_config = store_tests_swift.read_config(config_path) - - try: - self.cinder_config = parse_config(raw_config) - ret = store_tests_swift.keystone_authenticate( - self.cinder_config['test_cinder_store_auth_address'], - self.cinder_config['test_cinder_store_auth_version'], - self.cinder_config['test_cinder_store_tenant'], - self.cinder_config['test_cinder_store_user'], - self.cinder_config['test_cinder_store_key']) - (tenant_id, auth_token, service_catalog) = ret - self.context = glance.context.RequestContext( - tenant=tenant_id, - service_catalog=service_catalog, - auth_tok=auth_token) - self.cinder_client = cinder.get_cinderclient(self.context) - except Exception as e: - msg = "Cinder backend isn't set up: %s" % e - self.skipTest(msg) - - super(TestCinderStore, self).setUp() - - def get_store(self, **kwargs): - store = cinder.Store(context=kwargs.get('context') or self.context) - return store - - def stash_image(self, image_id, image_data): - #(zhiyan): Currently cinder store is a partial implementation, - # after Cinder expose 'brick' library, 'host-volume-attaching' and - # 'multiple-attaching' enhancement ready, the store will support - # ADD/GET/DELETE interface. - raise NotImplementedError('stash_image can not be implemented so far') diff --git a/glance/tests/functional/store/test_filesystem.py b/glance/tests/functional/store/test_filesystem.py deleted file mode 100644 index 19a0a3f0ed..0000000000 --- a/glance/tests/functional/store/test_filesystem.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2012 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -""" -Functional tests for the File store interface -""" - -import os -import os.path - -import fixtures -import oslo.config.cfg -import testtools - -import glance.store.filesystem -import glance.tests.functional.store as store_tests - - -class TestFilesystemStore(store_tests.BaseTestCase, testtools.TestCase): - - store_cls_path = 'glance.store.filesystem.Store' - store_cls = glance.store.filesystem.Store - store_name = 'filesystem' - - def setUp(self): - super(TestFilesystemStore, self).setUp() - self.tmp_dir = self.useFixture(fixtures.TempDir()).path - - self.store_dir = os.path.join(self.tmp_dir, 'images') - os.mkdir(self.store_dir) - - config_file = os.path.join(self.tmp_dir, 'glance.conf') - with open(config_file, 'w') as fap: - fap.write("[DEFAULT]\n") - fap.write("filesystem_store_datadir=%s" % self.store_dir) - - oslo.config.cfg.CONF(default_config_files=[config_file], args=[]) - - def get_store(self, **kwargs): - store = glance.store.filesystem.Store(context=kwargs.get('context')) - store.configure() - store.configure_add() - return store - - def stash_image(self, image_id, image_data): - filepath = os.path.join(self.store_dir, image_id) - with open(filepath, 'w') as fap: - fap.write(image_data) - return 'file://%s' % filepath diff --git a/glance/tests/functional/store/test_gridfs.py b/glance/tests/functional/store/test_gridfs.py deleted file mode 100644 index 94fc4dac72..0000000000 --- a/glance/tests/functional/store/test_gridfs.py +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright 2013 Red Hat, Inc -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -""" -Functional tests for the gridfs store interface - -Set the GLANCE_TEST_GRIDFS_CONF environment variable to the location -of a Glance config that defines how to connect to a functional -GridFS backend -""" - -import ConfigParser -import os - -import oslo.config.cfg -import testtools - -import glance.store.gridfs -import glance.tests.functional.store as store_tests - - -try: - import gridfs - import pymongo -except ImportError: - gridfs = None - - -def read_config(path): - cp = ConfigParser.RawConfigParser() - cp.read(path) - return cp - - -def parse_config(config): - out = {} - options = [ - 'mongodb_store_db', - 'mongodb_store_uri'] - - for option in options: - out[option] = config.defaults()[option] - - return out - - -class TestGridfsStore(store_tests.BaseTestCase, testtools.TestCase): - - store_cls_path = 'glance.store.gridfs.Store' - store_cls = glance.store.gridfs.Store - store_name = 'gridfs' - - def setUp(self): - config_path = os.environ.get('GLANCE_TEST_GRIDFS_CONF') - if not config_path or not gridfs: - msg = "GLANCE_TEST_GRIDFS_CONF environ not set." - self.skipTest(msg) - - oslo.config.cfg.CONF(args=[], default_config_files=[config_path]) - - raw_config = read_config(config_path) - self.gfs_config = parse_config(raw_config) - super(TestGridfsStore, self).setUp() - - def get_store(self, **kwargs): - store = self.store_cls(context=kwargs.get('context')) - store.configure() - store.configure_add() - return store - - def stash_image(self, image_id, image_data): - conn = pymongo.MongoClient(self.gfs_config.get("mongodb_store_uri")) - fs = gridfs.GridFS(conn[self.gfs_config.get("mongodb_store_db")]) - fs.put(image_data, _id=image_id) - return 'gridfs://%s' % image_id diff --git a/glance/tests/functional/store/test_http.py b/glance/tests/functional/store/test_http.py deleted file mode 100644 index c447bc0dfd..0000000000 --- a/glance/tests/functional/store/test_http.py +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright 2012 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -""" -Functional tests for the File store interface -""" - -import BaseHTTPServer -import os -import signal -import testtools - -import glance.store.http -import glance.tests.functional.store as store_tests - - -def get_handler_class(fixture): - class StaticHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): - def do_GET(self): - self.send_response(200) - self.send_header('Content-Length', str(len(fixture))) - self.end_headers() - self.wfile.write(fixture) - return - - def do_HEAD(self): - self.send_response(200) - self.send_header('Content-Length', str(len(fixture))) - self.end_headers() - return - - def log_message(*args, **kwargs): - # Override this method to prevent debug output from going - # to stderr during testing - return - - return StaticHTTPRequestHandler - - -def http_server(image_id, image_data): - server_address = ('127.0.0.1', 0) - handler_class = get_handler_class(image_data) - httpd = BaseHTTPServer.HTTPServer(server_address, handler_class) - port = httpd.socket.getsockname()[1] - - pid = os.fork() - if pid == 0: - httpd.serve_forever() - else: - return pid, port - - -class TestHTTPStore(store_tests.BaseTestCase, testtools.TestCase): - - store_cls_path = 'glance.store.http.Store' - store_cls = glance.store.http.Store - store_name = 'http' - - def setUp(self): - super(TestHTTPStore, self).setUp() - self.kill_pid = None - - def tearDown(self): - if self.kill_pid is not None: - os.kill(self.kill_pid, signal.SIGKILL) - - super(TestHTTPStore, self).tearDown() - - def get_store(self, **kwargs): - store = glance.store.http.Store(context=kwargs.get('context')) - store.configure() - return store - - def stash_image(self, image_id, image_data): - self.kill_pid, http_port = http_server(image_id, image_data) - return 'http://127.0.0.1:%s/' % (http_port,) diff --git a/glance/tests/functional/store/test_rbd.py b/glance/tests/functional/store/test_rbd.py deleted file mode 100644 index 99a596d0e3..0000000000 --- a/glance/tests/functional/store/test_rbd.py +++ /dev/null @@ -1,162 +0,0 @@ -# Copyright 2013 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -""" -Functional tests for the RBD store interface. - -Set the GLANCE_TEST_RBD_CONF environment variable to the location -of a Glance config that defines how to connect to a functional -RBD backend. This backend must be running Ceph Bobtail (0.56) or later. -""" - -import ConfigParser -import os -import uuid - -import oslo.config.cfg -import six -import testtools - -from glance.common import exception - -import glance.store.rbd -import glance.tests.functional.store as store_tests - -try: - import rados - import rbd -except ImportError: - rados = None - - -def read_config(path): - cp = ConfigParser.RawConfigParser() - cp.read(path) - return cp - - -def parse_config(config): - out = {} - options = [ - 'rbd_store_chunk_size', - 'rbd_store_pool', - 'rbd_store_user', - 'rbd_store_ceph_conf', - ] - - for option in options: - out[option] = config.defaults()[option] - - return out - - -class TestRBDStore(store_tests.BaseTestCase, testtools.TestCase): - - store_cls_path = 'glance.store.rbd.Store' - store_cls = glance.store.rbd.Store - store_name = 'rbd' - - def setUp(self): - config_path = os.environ.get('GLANCE_TEST_RBD_CONF') - if not config_path: - msg = "GLANCE_TEST_RBD_CONF environ not set." - self.skipTest(msg) - - oslo.config.cfg.CONF(args=[], default_config_files=[config_path]) - - raw_config = read_config(config_path) - config = parse_config(raw_config) - - if rados is None: - self.skipTest("rados python library not found") - - rados_client = rados.Rados(conffile=config['rbd_store_ceph_conf'], - rados_id=config['rbd_store_user']) - try: - rados_client.connect() - except rados.Error as e: - self.skipTest("Failed to connect to RADOS: %s" % e) - - try: - rados_client.create_pool(config['rbd_store_pool']) - except rados.Error as e: - rados_client.shutdown() - self.skipTest("Failed to create pool: %s") - - self.rados_client = rados_client - self.rbd_config = config - - super(TestRBDStore, self).setUp() - - def tearDown(self): - self.rados_client.delete_pool(self.rbd_config['rbd_store_pool']) - self.rados_client.shutdown() - - super(TestRBDStore, self).tearDown() - - def get_store(self, **kwargs): - store = glance.store.rbd.Store(context=kwargs.get('context')) - return store - - def stash_image(self, image_id, image_data): - fsid = self.rados_client.get_fsid() - pool = self.rbd_config['rbd_store_pool'] - librbd = rbd.RBD() - # image_id must not be unicode since librbd doesn't handle it - image_id = str(image_id) - snap_name = 'snap' - with self.rados_client.open_ioctx(pool) as ioctx: - librbd.create(ioctx, image_id, len(image_data), old_format=False, - features=rbd.RBD_FEATURE_LAYERING) - with rbd.Image(ioctx, image_id) as image: - image.write(image_data, 0) - image.create_snap(snap_name) - - return 'rbd://%s/%s/%s/%s' % (fsid, pool, image_id, snap_name) - - def test_unicode(self): - # librbd does not handle unicode, so make sure - # all paths through the rbd store convert a unicode image id - # and uri to ascii before passing it to librbd. - store = self.get_store() - - image_id = six.text_type(str(uuid.uuid4())) - image_size = 300 - image_data = six.StringIO('X' * image_size) - image_checksum = '41757066eaff7c4c6c965556b4d3c6c5' - - uri, add_size, add_checksum = store.add(image_id, - image_data, - image_size) - uri = six.text_type(uri) - - self.assertEqual(image_size, add_size) - self.assertEqual(image_checksum, add_checksum) - - location = glance.store.location.Location( - self.store_name, - store.get_store_location_class(), - uri=uri, - image_id=image_id) - - self.assertEqual(image_size, store.get_size(location)) - - get_iter, get_size = store.get(location) - - self.assertEqual(image_size, get_size) - self.assertEqual('X' * image_size, ''.join(get_iter)) - - store.delete(location) - - self.assertRaises(exception.NotFound, store.get, location) diff --git a/glance/tests/functional/store/test_s3.py b/glance/tests/functional/store/test_s3.py deleted file mode 100644 index cb16c4e829..0000000000 --- a/glance/tests/functional/store/test_s3.py +++ /dev/null @@ -1,124 +0,0 @@ -# Copyright 2012 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -""" -Functional tests for the S3 store interface - -Set the GLANCE_TEST_S3_CONF environment variable to the location -of a Glance config that defines how to connect to a functional -S3 backend -""" - -import ConfigParser -import os -import os.path - -import oslo.config.cfg -import six.moves.urllib.parse as urlparse -import testtools - -import glance.store.s3 -import glance.tests.functional.store as store_tests - -try: - from boto.s3.connection import S3Connection -except ImportError: - S3Connection = None - - -def read_config(path): - cp = ConfigParser.RawConfigParser() - cp.read(path) - return cp - - -def parse_config(config): - out = {} - options = [ - 's3_store_host', - 's3_store_access_key', - 's3_store_secret_key', - 's3_store_bucket', - 's3_store_bucket_url_format', - ] - - for option in options: - out[option] = config.defaults()[option] - - return out - - -def s3_connect(s3_host, access_key, secret_key, calling_format): - return S3Connection(access_key, secret_key, host=s3_host, - is_secure=False, calling_format=calling_format) - - -def s3_put_object(s3_client, bucket_name, object_name, contents): - bucket = s3_client.get_bucket(bucket_name) - key = bucket.new_key(object_name) - key.set_contents_from_string(contents) - - -class TestS3Store(store_tests.BaseTestCase, testtools.TestCase): - - store_cls_path = 'glance.store.s3.Store' - store_cls = glance.store.s3.Store - store_name = 's3' - - def setUp(self): - config_path = os.environ.get('GLANCE_TEST_S3_CONF') - if not config_path: - msg = "GLANCE_TEST_S3_CONF environ not set." - self.skipTest(msg) - - oslo.config.cfg.CONF(args=[], default_config_files=[config_path]) - - raw_config = read_config(config_path) - config = parse_config(raw_config) - - calling_format = glance.store.s3.get_calling_format( - config['s3_store_bucket_url_format']) - - s3_client = s3_connect(config['s3_store_host'], - config['s3_store_access_key'], - config['s3_store_secret_key'], - calling_format) - - #NOTE(bcwaldon): ensure we have a functional S3 connection - s3_client.get_all_buckets() - - self.s3_client = s3_client - self.s3_config = config - - super(TestS3Store, self).setUp() - - def get_store(self, **kwargs): - store = glance.store.s3.Store(context=kwargs.get('context')) - store.configure() - store.configure_add() - return store - - def stash_image(self, image_id, image_data): - bucket_name = self.s3_config['s3_store_bucket'] - s3_put_object(self.s3_client, bucket_name, image_id, 'XXX') - - s3_store_host = urlparse.urlparse(self.s3_config['s3_store_host']) - access_key = urlparse.quote(self.s3_config['s3_store_access_key']) - secret_key = self.s3_config['s3_store_secret_key'] - auth_chunk = '%s:%s' % (access_key, secret_key) - netloc = '%s@%s' % (auth_chunk, s3_store_host.netloc) - path = os.path.join(s3_store_host.path, bucket_name, image_id) - - # This is an s3 url with // on the end - return 's3://%s%s' % (netloc, path) diff --git a/glance/tests/functional/store/test_sheepdog.py b/glance/tests/functional/store/test_sheepdog.py deleted file mode 100644 index 22d62d6e83..0000000000 --- a/glance/tests/functional/store/test_sheepdog.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright 2013 Taobao Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -""" -Functional tests for the Sheepdog store interface -""" - -import os -import os.path - -import fixtures -import oslo.config.cfg -import testtools - -from glance.store import BackendException -import glance.store.sheepdog as sheepdog -import glance.tests.functional.store as store_tests -import glance.tests.utils - - -class TestSheepdogStore(store_tests.BaseTestCase, testtools.TestCase): - - store_cls_path = 'glance.store.sheepdog.Store' - store_cls = glance.store.sheepdog.Store - store_name = 'sheepdog' - - def setUp(self): - image = sheepdog.SheepdogImage(sheepdog.DEFAULT_ADDR, - sheepdog.DEFAULT_PORT, - "test", - sheepdog.DEFAULT_CHUNKSIZE) - try: - image.create(512) - except BackendException: - msg = "Sheepdog cluster isn't set up" - self.skipTest(msg) - image.delete() - - self.tmp_dir = self.useFixture(fixtures.TempDir()).path - - config_file = os.path.join(self.tmp_dir, 'glance.conf') - with open(config_file, 'w') as f: - f.write("[DEFAULT]\n") - f.write("default_store = sheepdog") - - oslo.config.cfg.CONF(default_config_files=[config_file], args=[]) - super(TestSheepdogStore, self).setUp() - - def get_store(self, **kwargs): - store = sheepdog.Store(context=kwargs.get('context')) - return store - - def stash_image(self, image_id, image_data): - image_size = len(image_data) - image = sheepdog.SheepdogImage(sheepdog.DEFAULT_ADDR, - sheepdog.DEFAULT_PORT, - image_id, - sheepdog.DEFAULT_CHUNKSIZE) - image.create(image_size) - total = left = image_size - while left > 0: - length = min(sheepdog.DEFAULT_CHUNKSIZE, left) - image.write(image_data, total - left, length) - left -= length - - return 'sheepdog://%s' % image_id diff --git a/glance/tests/functional/store/test_swift.py b/glance/tests/functional/store/test_swift.py deleted file mode 100644 index 524f11ec40..0000000000 --- a/glance/tests/functional/store/test_swift.py +++ /dev/null @@ -1,535 +0,0 @@ -# Copyright 2012 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -""" -Functional tests for the Swift store interface - -Set the GLANCE_TEST_SWIFT_CONF environment variable to the location -of a Glance config that defines how to connect to a functional -Swift backend -""" - -import ConfigParser -import hashlib -import os -import os.path -import random -import string -import uuid - -import oslo.config.cfg -import six -import six.moves.urllib.parse as urlparse -import testtools - -from glance.common import exception -import glance.common.utils as common_utils - -import glance.store.swift -import glance.tests.functional.store as store_tests - -try: - import swiftclient -except ImportError: - swiftclient = None - - -class SwiftStoreError(RuntimeError): - pass - - -def _uniq(value): - return '%s.%d' % (value, random.randint(0, 99999)) - - -def read_config(path): - cp = ConfigParser.RawConfigParser() - cp.read(path) - return cp - - -def parse_config(config): - out = {} - options = [ - 'swift_store_auth_address', - 'swift_store_auth_version', - 'swift_store_user', - 'swift_store_key', - 'swift_store_container', - ] - - for option in options: - out[option] = config.defaults()[option] - - return out - - -def swift_connect(auth_url, auth_version, user, key): - try: - return swiftclient.Connection(authurl=auth_url, - auth_version=auth_version, - user=user, - key=key, - snet=False, - retries=1) - except AttributeError: - raise SwiftStoreError("Could not find swiftclient module") - - -def swift_list_containers(swift_conn): - try: - _, containers = swift_conn.get_account() - except Exception as e: - msg = ("Failed to list containers (get_account) " - "from Swift. Got error: %s" % e) - raise SwiftStoreError(msg) - else: - return containers - - -def swift_create_container(swift_conn, container_name): - try: - swift_conn.put_container(container_name) - except swiftclient.ClientException as e: - msg = "Failed to create container. Got error: %s" % e - raise SwiftStoreError(msg) - - -def swift_get_container(swift_conn, container_name, **kwargs): - return swift_conn.get_container(container_name, **kwargs) - - -def swift_delete_container(swift_conn, container_name): - try: - swift_conn.delete_container(container_name) - except swiftclient.ClientException as e: - msg = "Failed to delete container from Swift. Got error: %s" % e - raise SwiftStoreError(msg) - - -def swift_put_object(swift_conn, container_name, object_name, contents): - return swift_conn.put_object(container_name, object_name, contents) - - -def swift_head_object(swift_conn, container_name, obj_name): - return swift_conn.head_object(container_name, obj_name) - - -def keystone_authenticate(auth_url, auth_version, tenant_name, - username, password): - assert int(auth_version) == 2, 'Only auth version 2 is supported' - - import keystoneclient.v2_0.client - ksclient = keystoneclient.v2_0.client.Client(tenant_name=tenant_name, - username=username, - password=password, - auth_url=auth_url) - - auth_resp = ksclient.service_catalog.catalog - tenant_id = auth_resp['token']['tenant']['id'] - service_catalog = auth_resp['serviceCatalog'] - return tenant_id, ksclient.auth_token, service_catalog - - -class TestSwiftStore(store_tests.BaseTestCase, testtools.TestCase): - - store_cls_path = 'glance.store.swift.Store' - store_cls = glance.store.swift.Store - store_name = 'swift' - - def setUp(self): - config_path = os.environ.get('GLANCE_TEST_SWIFT_CONF') - if not config_path: - msg = "GLANCE_TEST_SWIFT_CONF environ not set." - self.skipTest(msg) - - oslo.config.cfg.CONF(args=[], default_config_files=[config_path]) - - raw_config = read_config(config_path) - config = parse_config(raw_config) - - swift = swift_connect(config['swift_store_auth_address'], - config['swift_store_auth_version'], - config['swift_store_user'], - config['swift_store_key']) - - #NOTE(bcwaldon): Ensure we have a functional swift connection - swift_list_containers(swift) - - self.swift_client = swift - self.swift_config = config - - self.swift_config['swift_store_create_container_on_put'] = True - - super(TestSwiftStore, self).setUp() - - def get_store(self, **kwargs): - store = glance.store.swift.Store(context=kwargs.get('context')) - return store - - def test_object_chunking(self): - """Upload an image that is split into multiple swift objects. - - We specifically check the case that - image_size % swift_store_large_object_chunk_size != 0 to - ensure we aren't losing image data. - """ - self.config( - swift_store_large_object_size=2, # 2 MB - swift_store_large_object_chunk_size=2, # 2 MB - ) - store = self.get_store() - image_id = str(uuid.uuid4()) - image_size = 5242880 # 5 MB - image_data = six.StringIO('X' * image_size) - image_checksum = 'eb7f8c3716b9f059cee7617a4ba9d0d3' - uri, add_size, add_checksum, _ = store.add(image_id, - image_data, - image_size) - - self.assertEqual(image_size, add_size) - self.assertEqual(image_checksum, add_checksum) - - location = glance.store.location.Location( - self.store_name, - store.get_store_location_class(), - uri=uri, - image_id=image_id) - - # Store interface should still be respected even though - # we are storing images in multiple Swift objects - (get_iter, get_size) = store.get(location) - self.assertEqual(5242880, get_size) - self.assertEqual('X' * 5242880, ''.join(get_iter)) - - # The object should have a manifest pointing to the chunks - # of image data - swift_location = location.store_location - headers = swift_head_object(self.swift_client, - swift_location.container, - swift_location.obj) - manifest = headers.get('x-object-manifest') - self.assertTrue(manifest) - - # Verify the objects in the manifest exist - manifest_container, manifest_prefix = manifest.split('/', 1) - container = swift_get_container(self.swift_client, - manifest_container, - prefix=manifest_prefix) - segments = [segment['name'] for segment in container[1]] - - for segment in segments: - headers = swift_head_object(self.swift_client, - manifest_container, - segment) - self.assertTrue(headers.get('content-length')) - - # Since we used a 5 MB image with a 2 MB chunk size, we should - # expect to see three data objects - self.assertEqual(3, len(segments), 'Got segments %s' % segments) - - # Add an object that should survive the delete operation - non_image_obj = image_id + '0' - swift_put_object(self.swift_client, - manifest_container, - non_image_obj, - 'XXX') - - store.delete(location) - - # Verify the segments in the manifest are all gone - for segment in segments: - self.assertRaises(swiftclient.ClientException, - swift_head_object, - self.swift_client, - manifest_container, - segment) - - # Verify the manifest is gone too - self.assertRaises(swiftclient.ClientException, - swift_head_object, - self.swift_client, - manifest_container, - swift_location.obj) - - # Verify that the non-image object was not deleted - headers = swift_head_object(self.swift_client, - manifest_container, - non_image_obj) - self.assertTrue(headers.get('content-length')) - - # Clean up - self.swift_client.delete_object(manifest_container, - non_image_obj) - - # Simulate exceeding 'image_size_cap' setting - image_data = six.StringIO('X' * image_size) - image_data = common_utils.LimitingReader(image_data, image_size - 1) - image_id = str(uuid.uuid4()) - self.assertRaises(exception.ImageSizeLimitExceeded, - store.add, - image_id, - image_data, - image_size) - - # Verify written segments have been deleted - container = swift_get_container(self.swift_client, - manifest_container, - prefix=image_id) - segments = [segment['name'] for segment in container[1]] - self.assertEqual(0, len(segments), 'Got segments %s' % segments) - - def test_retries_fail_start_of_download(self): - """ - Get an object from Swift where Swift does not complete the request - in one attempt. Fails at the start of the download. - """ - self.config( - swift_store_retry_get_count=1, - ) - store = self.get_store() - image_id = str(uuid.uuid4()) - image_size = 1024 * 1024 * 5 # 5 MB - chars = string.ascii_uppercase + string.digits - image_data = ''.join(random.choice(chars) for x in range(image_size)) - image_checksum = hashlib.md5(image_data) - uri, add_size, add_checksum, _ = store.add(image_id, - image_data, - image_size) - - location = glance.store.location.Location( - self.store_name, - store.get_store_location_class(), - uri=uri, - image_id=image_id) - - def iter_wrapper(iterable): - # raise StopIteration as soon as iteration begins - yield '' - - (get_iter, get_size) = store.get(location) - get_iter.wrapped = glance.store.swift.swift_retry_iter( - iter_wrapper(get_iter.wrapped), image_size, - store, location.store_location) - self.assertEqual(image_size, get_size) - received_data = ''.join(get_iter.wrapped) - self.assertEqual(image_data, received_data) - self.assertEqual(image_checksum.hexdigest(), - hashlib.md5(received_data).hexdigest()) - - def test_retries_fail_partway_through_download(self): - """ - Get an object from Swift where Swift does not complete the request - in one attempt. Fails partway through the download. - """ - self.config( - swift_store_retry_get_count=1, - ) - store = self.get_store() - image_id = str(uuid.uuid4()) - image_size = 1024 * 1024 * 5 # 5 MB - chars = string.ascii_uppercase + string.digits - image_data = ''.join(random.choice(chars) for x in range(image_size)) - image_checksum = hashlib.md5(image_data) - uri, add_size, add_checksum, _ = store.add(image_id, - image_data, - image_size) - - location = glance.store.location.Location( - self.store_name, - store.get_store_location_class(), - uri=uri, - image_id=image_id) - - def iter_wrapper(iterable): - bytes_received = 0 - for chunk in iterable: - yield chunk - bytes_received += len(chunk) - if bytes_received > (image_size / 2): - raise StopIteration - - (get_iter, get_size) = store.get(location) - get_iter.wrapped = glance.store.swift.swift_retry_iter( - iter_wrapper(get_iter.wrapped), image_size, - store, location.store_location) - self.assertEqual(image_size, get_size) - received_data = ''.join(get_iter.wrapped) - self.assertEqual(image_data, received_data) - self.assertEqual(image_checksum.hexdigest(), - hashlib.md5(received_data).hexdigest()) - - def test_retries_fail_end_of_download(self): - """ - Get an object from Swift where Swift does not complete the request - in one attempt. Fails at the end of the download - """ - self.config( - swift_store_retry_get_count=1, - ) - store = self.get_store() - image_id = str(uuid.uuid4()) - image_size = 1024 * 1024 * 5 # 5 MB - chars = string.ascii_uppercase + string.digits - image_data = ''.join(random.choice(chars) for x in range(image_size)) - image_checksum = hashlib.md5(image_data) - uri, add_size, add_checksum, _ = store.add(image_id, - image_data, - image_size) - - location = glance.store.location.Location( - self.store_name, - store.get_store_location_class(), - uri=uri, - image_id=image_id) - - def iter_wrapper(iterable): - bytes_received = 0 - for chunk in iterable: - yield chunk - bytes_received += len(chunk) - if bytes_received == image_size: - raise StopIteration - - (get_iter, get_size) = store.get(location) - get_iter.wrapped = glance.store.swift.swift_retry_iter( - iter_wrapper(get_iter.wrapped), image_size, - store, location.store_location) - self.assertEqual(image_size, get_size) - received_data = ''.join(get_iter.wrapped) - self.assertEqual(image_data, received_data) - self.assertEqual(image_checksum.hexdigest(), - hashlib.md5(received_data).hexdigest()) - - def stash_image(self, image_id, image_data): - container_name = self.swift_config['swift_store_container'] - swift_put_object(self.swift_client, - container_name, - image_id, - 'XXX') - - #NOTE(bcwaldon): This is a hack until we find a better way to - # build this URL - auth_url = self.swift_config['swift_store_auth_address'] - auth_url = urlparse.urlparse(auth_url) - user = urlparse.quote(self.swift_config['swift_store_user']) - key = self.swift_config['swift_store_key'] - netloc = ''.join(('%s:%s' % (user, key), '@', auth_url.netloc)) - path = os.path.join(auth_url.path, container_name, image_id) - - # This is an auth url with // on the end - return 'swift+http://%s%s' % (netloc, path) - - def test_multitenant(self): - """Ensure an image is properly configured when using multitenancy.""" - self.config( - swift_store_multi_tenant=True, - ) - - swift_store_user = self.swift_config['swift_store_user'] - tenant_name, username = swift_store_user.split(':') - tenant_id, auth_token, service_catalog = keystone_authenticate( - self.swift_config['swift_store_auth_address'], - self.swift_config['swift_store_auth_version'], - tenant_name, - username, - self.swift_config['swift_store_key']) - - context = glance.context.RequestContext( - tenant=tenant_id, - service_catalog=service_catalog, - auth_tok=auth_token) - store = self.get_store(context=context) - - image_id = str(uuid.uuid4()) - image_data = six.StringIO('XXX') - uri, _, _, _ = store.add(image_id, image_data, 3) - - location = glance.store.location.Location( - self.store_name, - store.get_store_location_class(), - uri=uri, - image_id=image_id) - - read_tenant = str(uuid.uuid4()) - write_tenant = str(uuid.uuid4()) - store.set_acls(location, - public=False, - read_tenants=[read_tenant], - write_tenants=[write_tenant]) - - container_name = location.store_location.container - container, _ = swift_get_container(self.swift_client, container_name) - self.assertEqual(read_tenant + ':*', - container.get('x-container-read')) - self.assertEqual(write_tenant + ':*', - container.get('x-container-write')) - - store.set_acls(location, public=True, read_tenants=[read_tenant]) - - container_name = location.store_location.container - container, _ = swift_get_container(self.swift_client, container_name) - self.assertEqual('.r:*,.rlistings', container.get('x-container-read')) - self.assertEqual('', container.get('x-container-write', '')) - - (get_iter, get_size) = store.get(location) - self.assertEqual(3, get_size) - self.assertEqual('XXX', ''.join(get_iter)) - - store.delete(location) - - def test_delayed_delete_with_auth(self): - """Ensure delete works with delayed delete and auth - - Reproduces LP bug 1238604. - """ - self.config( - scrubber_datadir="/tmp", - ) - swift_store_user = self.swift_config['swift_store_user'] - tenant_name, username = swift_store_user.split(':') - tenant_id, auth_token, service_catalog = keystone_authenticate( - self.swift_config['swift_store_auth_address'], - self.swift_config['swift_store_auth_version'], - tenant_name, - username, - self.swift_config['swift_store_key']) - - context = glance.context.RequestContext( - tenant=tenant_id, - service_catalog=service_catalog, - auth_tok=auth_token) - store = self.get_store(context=context) - - image_id = str(uuid.uuid4()) - image_data = six.StringIO('data') - uri, _, _, _ = store.add(image_id, image_data, 4) - - location = glance.store.location.Location( - self.store_name, - store.get_store_location_class(), - uri=uri, - image_id=image_id) - - container_name = location.store_location.container - container, _ = swift_get_container(self.swift_client, container_name) - - (get_iter, get_size) = store.get(location) - self.assertEqual(4, get_size) - self.assertEqual('data', ''.join(get_iter)) - - glance.store.schedule_delayed_delete_from_backend(context, - uri, - image_id) - store.delete(location) diff --git a/glance/tests/functional/store/test_vmware_datastore.py b/glance/tests/functional/store/test_vmware_datastore.py deleted file mode 100644 index 8899759d5f..0000000000 --- a/glance/tests/functional/store/test_vmware_datastore.py +++ /dev/null @@ -1,176 +0,0 @@ -# Copyright 2014 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Functional tests for the VMware Datastore store interface - -Set the GLANCE_TEST_VMWARE_CONF environment variable to the location -of a Glance config that defines how to connect to a functional -VMware Datastore backend -""" - -import ConfigParser -import httplib -import logging -import os -import uuid - -import oslo.config.cfg -from oslo.vmware import api -import six -import six.moves.urllib.parse as urlparse -import testtools - -from glance.common import exception -import glance.store.location -import glance.store.vmware_datastore as vm_store -import glance.tests.functional.store as store_tests - - -logging.getLogger('suds').setLevel(logging.INFO) - - -def read_config(path): - cp = ConfigParser.RawConfigParser() - cp.read(path) - return cp - - -def parse_config(config): - out = {} - options = [ - 'vmware_server_host', - 'vmware_server_username', - 'vmware_server_password', - 'vmware_api_retry_count', - 'vmware_task_poll_interval', - 'vmware_store_image_dir', - 'vmware_datacenter_path', - 'vmware_datastore_name', - 'vmware_api_insecure', - ] - for option in options: - out[option] = config.defaults()[option] - - return out - - -class VMwareDatastoreStoreError(RuntimeError): - pass - - -def vsphere_connect(server_ip, server_username, server_password, - api_retry_count, task_poll_interval, - scheme='https', create_session=True, wsdl_loc=None): - try: - return api.VMwareAPISession(server_ip, - server_username, - server_password, - api_retry_count, - task_poll_interval, - scheme=scheme, - create_session=create_session, - wsdl_loc=wsdl_loc) - except AttributeError: - raise VMwareDatastoreStoreError( - 'Could not find VMware datastore module') - - -class TestVMwareDatastoreStore(store_tests.BaseTestCase, testtools.TestCase): - - store_cls_path = 'glance.store.vmware_datastore.Store' - store_cls = vm_store.Store - store_name = 'vmware_datastore' - - def _build_vim_cookie_header(self, vim_cookies): - """Build ESX host session cookie header.""" - if len(list(vim_cookies)) > 0: - cookie = list(vim_cookies)[0] - return cookie.name + '=' + cookie.value - - def setUp(self): - config_path = os.environ.get('GLANCE_TEST_VMWARE_CONF') - if not config_path: - msg = 'GLANCE_TEST_VMWARE_CONF environ not set.' - self.skipTest(msg) - - oslo.config.cfg.CONF(args=[], default_config_files=[config_path]) - - raw_config = read_config(config_path) - config = parse_config(raw_config) - scheme = 'http' if config['vmware_api_insecure'] == 'True' else 'https' - self.vsphere = vsphere_connect(config['vmware_server_host'], - config['vmware_server_username'], - config['vmware_server_password'], - config['vmware_api_retry_count'], - config['vmware_task_poll_interval'], - scheme=scheme) - - self.vmware_config = config - super(TestVMwareDatastoreStore, self).setUp() - - def get_store(self, **kwargs): - store = vm_store.Store(context=kwargs.get('context')) - return store - - def stash_image(self, image_id, image_data): - server_ip = self.vmware_config['vmware_server_host'] - path = os.path.join( - vm_store.DS_URL_PREFIX, - self.vmware_config['vmware_store_image_dir'].strip('/'), image_id) - dc_path = self.vmware_config.get('vmware_datacenter_path', - 'ha-datacenter') - param_list = {'dcPath': dc_path, - 'dsName': self.vmware_config['vmware_datastore_name']} - query = urlparse.urlencode(param_list) - conn = (httplib.HTTPConnection(server_ip) - if self.vmware_config['vmware_api_insecure'] == 'True' - else httplib.HTTPSConnection(server_ip)) - cookie = self._build_vim_cookie_header( - self.vsphere.vim.client.options.transport.cookiejar) - headers = {'Cookie': cookie, 'Content-Length': len(image_data)} - url = urlparse.quote('%s?%s' % (path, query)) - conn.request('PUT', url, image_data, headers) - conn.getresponse() - - return '%s://%s%s?%s' % (vm_store.STORE_SCHEME, server_ip, path, query) - - def test_timeout(self): - store = self.get_store() - store._session.logout() - image_id = str(uuid.uuid4()) - image_data = six.StringIO('XXX') - image_checksum = 'bc9189406be84ec297464a514221406d' - uri, add_size, add_checksum, _ = store.add(image_id, image_data, 3) - self.assertEqual(3, add_size) - self.assertEqual(image_checksum, add_checksum) - - loc = glance.store.location.Location( - self.store_name, - store.get_store_location_class(), - uri=uri, - image_id=image_id) - store._session.logout() - get_iter, get_size = store.get(loc) - self.assertEqual(3, get_size) - self.assertEqual('XXX', ''.join(get_iter)) - - store._session.logout() - image_size = store.get_size(loc) - self.assertEqual(3, image_size) - - store._session.logout() - store.delete(loc) - self.assertRaises(exception.NotFound, store.get, loc) diff --git a/glance/tests/functional/test_scrubber.py b/glance/tests/functional/test_scrubber.py index a92a04f8d8..ae4f9ee885 100644 --- a/glance/tests/functional/test_scrubber.py +++ b/glance/tests/functional/test_scrubber.py @@ -17,18 +17,14 @@ import os import sys import time +import glance_store.location import httplib2 from six.moves import xrange -import swiftclient from glance.common import crypt from glance.openstack.common import jsonutils from glance.openstack.common import units -from glance.store.swift import StoreLocation from glance.tests import functional -from glance.tests.functional.store.test_swift import parse_config -from glance.tests.functional.store.test_swift import read_config -from glance.tests.functional.store.test_swift import swift_connect from glance.tests.utils import execute @@ -133,85 +129,20 @@ class TestScrubber(functional.FunctionalTest): self.stop_servers() - def test_scrubber_app_against_swift(self): - """ - test that the glance-scrubber script runs successfully against a swift - backend when not in daemon mode - """ - config_path = os.environ.get('GLANCE_TEST_SWIFT_CONF') - if not config_path: - msg = "GLANCE_TEST_SWIFT_CONF environ not set." - self.skipTest(msg) - - raw_config = read_config(config_path) - swift_config = parse_config(raw_config) - - self.cleanup() - self.start_servers(delayed_delete=True, daemon=False, - metadata_encryption_key='', - default_store='swift', **swift_config) - - # add an image - headers = { - 'x-image-meta-name': 'test_image', - 'x-image-meta-is_public': 'true', - 'x-image-meta-disk_format': 'raw', - 'x-image-meta-container_format': 'ovf', - 'content-type': 'application/octet-stream', - } - path = "http://%s:%d/v1/images" % ("127.0.0.1", self.api_port) - http = httplib2.Http() - response, content = http.request(path, 'POST', body='XXX', - headers=headers) - # ensure the request was successful and the image is active - self.assertEqual(response.status, 201) - image = jsonutils.loads(content)['image'] - self.assertEqual('active', image['status']) - image_id = image['id'] - - # delete the image - path = "http://%s:%d/v1/images/%s" % ("127.0.0.1", self.api_port, - image_id) - http = httplib2.Http() - response, content = http.request(path, 'DELETE') - self.assertEqual(response.status, 200) - - # ensure the image is marked pending delete - response, content = http.request(path, 'HEAD') - self.assertEqual(response.status, 200) - self.assertEqual('pending_delete', response['x-image-meta-status']) - - # wait for the scrub time on the image to pass - time.sleep(self.api_server.scrub_time) - - # call the scrubber to scrub images - exe_cmd = "%s -m glance.cmd.scrubber" % sys.executable - cmd = ("%s --config-file %s" % - (exe_cmd, self.scrubber_daemon.conf_file_name)) - exitcode, out, err = execute(cmd, raise_error=False) - self.assertEqual(0, exitcode) - - # ensure the image has been successfully deleted - self.wait_for_scrub(path) - - self.stop_servers() - def test_scrubber_with_metadata_enc(self): """ test that files written to scrubber_data_dir use metadata_encryption_key when available to encrypt the location """ - config_path = os.environ.get('GLANCE_TEST_SWIFT_CONF') - if not config_path: - msg = "GLANCE_TEST_SWIFT_CONF environ not set." - self.skipTest(msg) - - raw_config = read_config(config_path) - swift_config = parse_config(raw_config) + # FIXME(flaper87): It looks like an older commit + # may have broken this test. The file_queue `add_location` + # is not being called. + self.skipTest("Test broken. See bug #1366682") self.cleanup() - self.start_servers(delayed_delete=True, daemon=True, - default_store='swift', **swift_config) + self.start_servers(delayed_delete=True, + daemon=True, + default_store='file') # add an image headers = { @@ -221,6 +152,7 @@ class TestScrubber(functional.FunctionalTest): 'x-image-meta-container_format': 'ovf', 'content-type': 'application/octet-stream', } + path = "http://%s:%d/v1/images" % ("127.0.0.1", self.api_port) http = httplib2.Http() response, content = http.request(path, 'POST', body='XXX', @@ -231,7 +163,8 @@ class TestScrubber(functional.FunctionalTest): image_id = image['id'] # delete the image - path = "http://%s:%d/v1/images/%s" % ("127.0.0.1", self.api_port, + path = "http://%s:%d/v1/images/%s" % ("127.0.0.1", + self.api_port, image_id) http = httplib2.Http() response, content = http.request(path, 'DELETE') @@ -252,86 +185,13 @@ class TestScrubber(functional.FunctionalTest): decrypted_uri = crypt.urlsafe_decrypt( self.api_server.metadata_encryption_key, marker_uri) - loc = StoreLocation({}) + loc = glance_store.location.StoreLocation({}) loc.parse_uri(decrypted_uri) - self.assertIn(loc.scheme, ("swift+http", "swift+https")) + self.assertEqual(loc.scheme, "file") self.assertEqual(image['id'], loc.obj) self.wait_for_scrub(path) - - self.stop_servers() - - def test_scrubber_handles_swift_missing(self): - """ - Test that the scrubber handles the case where the image to be scrubbed - is missing from swift - """ - config_path = os.environ.get('GLANCE_TEST_SWIFT_CONF') - if not config_path: - msg = "GLANCE_TEST_SWIFT_CONF environ not set." - self.skipTest(msg) - - raw_config = read_config(config_path) - swift_config = parse_config(raw_config) - - self.cleanup() - self.start_servers(delayed_delete=True, daemon=False, - default_store='swift', **swift_config) - - # add an image - headers = { - 'x-image-meta-name': 'test_image', - 'x-image-meta-is_public': 'true', - 'x-image-meta-disk_format': 'raw', - 'x-image-meta-container_format': 'ovf', - 'content-type': 'application/octet-stream', - } - path = "http://%s:%d/v1/images" % ("127.0.0.1", self.api_port) - http = httplib2.Http() - response, content = http.request(path, 'POST', body='XXX', - headers=headers) - self.assertEqual(response.status, 201) - image = jsonutils.loads(content)['image'] - self.assertEqual('active', image['status']) - image_id = image['id'] - - # delete the image - path = "http://%s:%d/v1/images/%s" % ("127.0.0.1", self.api_port, - image_id) - http = httplib2.Http() - response, content = http.request(path, 'DELETE') - self.assertEqual(response.status, 200) - - # ensure the image is marked pending delete - response, content = http.request(path, 'HEAD') - self.assertEqual(response.status, 200) - self.assertEqual('pending_delete', response['x-image-meta-status']) - - # go directly to swift and remove the image object - swift = swift_connect(swift_config['swift_store_auth_address'], - swift_config['swift_store_auth_version'], - swift_config['swift_store_user'], - swift_config['swift_store_key']) - swift.delete_object(swift_config['swift_store_container'], image_id) - try: - swift.head_object(swift_config['swift_store_container'], image_id) - self.fail('image should have been deleted from swift') - except swiftclient.ClientException as e: - self.assertEqual(e.http_status, 404) - - # wait for the scrub time on the image to pass - time.sleep(self.api_server.scrub_time) - - # run the scrubber app, and ensure it doesn't fall over - exe_cmd = "%s -m glance.cmd.scrubber" % sys.executable - cmd = ("%s --config-file %s" % - (exe_cmd, self.scrubber_daemon.conf_file_name)) - exitcode, out, err = execute(cmd, raise_error=False) - self.assertEqual(0, exitcode) - - self.wait_for_scrub(path) - self.stop_servers() def test_scrubber_delete_handles_exception(self): diff --git a/glance/tests/functional/v2/test_images.py b/glance/tests/functional/v2/test_images.py index 05222e6709..7f4721c015 100644 --- a/glance/tests/functional/v2/test_images.py +++ b/glance/tests/functional/v2/test_images.py @@ -13,6 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. +import BaseHTTPServer import os import signal import tempfile @@ -23,7 +24,6 @@ import six from glance.openstack.common import jsonutils from glance.tests import functional -from glance.tests.functional.store import test_http TENANT1 = str(uuid.uuid4()) @@ -32,6 +32,42 @@ TENANT3 = str(uuid.uuid4()) TENANT4 = str(uuid.uuid4()) +def get_handler_class(fixture): + class StaticHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header('Content-Length', str(len(fixture))) + self.end_headers() + self.wfile.write(fixture) + return + + def do_HEAD(self): + self.send_response(200) + self.send_header('Content-Length', str(len(fixture))) + self.end_headers() + return + + def log_message(*args, **kwargs): + # Override this method to prevent debug output from going + # to stderr during testing + return + + return StaticHTTPRequestHandler + + +def http_server(image_id, image_data): + server_address = ('127.0.0.1', 0) + handler_class = get_handler_class(image_data) + httpd = BaseHTTPServer.HTTPServer(server_address, handler_class) + port = httpd.socket.getsockname()[1] + + pid = os.fork() + if pid == 0: + httpd.serve_forever() + else: + return pid, port + + class TestImages(functional.FunctionalTest): def setUp(self): @@ -2289,7 +2325,7 @@ class TestImageLocationSelectionStrategy(functional.FunctionalTest): self.foo_image_file.write("foo image file") self.foo_image_file.flush() self.addCleanup(self.foo_image_file.close) - ret = test_http.http_server("foo_image_id", "foo_image") + ret = http_server("foo_image_id", "foo_image") self.http_server_pid, self.http_port = ret def tearDown(self): diff --git a/glance/tests/functional/v2/test_metadef_resourcetypes.py b/glance/tests/functional/v2/test_metadef_resourcetypes.py index 45c74da6f1..dc2ceefa75 100644 --- a/glance/tests/functional/v2/test_metadef_resourcetypes.py +++ b/glance/tests/functional/v2/test_metadef_resourcetypes.py @@ -33,7 +33,6 @@ import glance.notifier from glance.openstack.common import jsonutils as json import glance.openstack.common.log as logging import glance.schema -import glance.store LOG = logging.getLogger(__name__) _LE = i18n._LE diff --git a/glance/tests/integration/legacy_functional/base.py b/glance/tests/integration/legacy_functional/base.py index be4481f1d3..68fc302f37 100644 --- a/glance/tests/integration/legacy_functional/base.py +++ b/glance/tests/integration/legacy_functional/base.py @@ -15,6 +15,7 @@ import os.path import tempfile import fixtures +import glance_store from oslo.config import cfg from oslo.db import options @@ -23,7 +24,6 @@ from glance.common import config from glance.db import migration import glance.db.sqlalchemy.api import glance.registry.client.v1.client -import glance.store from glance import tests as glance_tests from glance.tests import utils as test_utils @@ -112,7 +112,6 @@ paste.filter_factory = glance.tests.utils:FakeAuthMiddleware.factory """ CONF = cfg.CONF -CONF.import_opt('filesystem_store_datadir', 'glance.store.filesystem') class ApiTest(test_utils.BaseTestCase): @@ -194,9 +193,14 @@ class ApiTest(test_utils.BaseTestCase): atexit.register(_delete_cached_db) def _setup_stores(self): + glance_store.register_opts(CONF) + glance_store.register_store_opts(CONF) + image_dir = os.path.join(self.test_dir, "images") - self.config(filesystem_store_datadir=image_dir) - glance.store.create_stores() + self.config(group='glance_store', + filesystem_store_datadir=image_dir) + + glance_store.create_stores() def _load_paste_app(self, name, flavor, conf): conf_file_path = os.path.join(self.test_dir, '%s-paste.ini' % name) diff --git a/glance/tests/integration/v2/base.py b/glance/tests/integration/v2/base.py index ddf2876f42..9938888743 100644 --- a/glance/tests/integration/v2/base.py +++ b/glance/tests/integration/v2/base.py @@ -18,6 +18,7 @@ import os.path import tempfile import fixtures +import glance_store from oslo.config import cfg from oslo.db import options @@ -26,7 +27,6 @@ from glance.common import config from glance.db import migration import glance.db.sqlalchemy.api import glance.registry.client.v1.client -import glance.store from glance import tests as glance_tests from glance.tests import utils as test_utils @@ -115,7 +115,6 @@ paste.filter_factory = glance.tests.utils:FakeAuthMiddleware.factory """ CONF = cfg.CONF -CONF.import_opt('filesystem_store_datadir', 'glance.store.filesystem') class ApiTest(test_utils.BaseTestCase): @@ -190,9 +189,14 @@ class ApiTest(test_utils.BaseTestCase): atexit.register(_delete_cached_db) def _setup_stores(self): + glance_store.register_opts(CONF) + glance_store.register_store_opts(CONF) + image_dir = os.path.join(self.test_dir, "images") - self.config(filesystem_store_datadir=image_dir) - glance.store.create_stores() + self.config(group='glance_store', + filesystem_store_datadir=image_dir) + + glance_store.create_stores() def _load_paste_app(self, name, flavor, conf): conf_file_path = os.path.join(self.test_dir, '%s-paste.ini' % name) diff --git a/glance/tests/unit/api/test_cmd.py b/glance/tests/unit/api/test_cmd.py index 510746adb8..46ee0a0cbe 100644 --- a/glance/tests/unit/api/test_cmd.py +++ b/glance/tests/unit/api/test_cmd.py @@ -13,6 +13,8 @@ import mock import sys +import glance_store as store +from oslo.config import cfg import six import glance.cmd.api @@ -26,6 +28,9 @@ import glance.image_cache.pruner from glance.tests import utils as test_utils +CONF = cfg.CONF + + class TestGlanceApiCmd(test_utils.BaseTestCase): __argv_backup = None @@ -45,6 +50,8 @@ class TestGlanceApiCmd(test_utils.BaseTestCase): self.stderr = six.StringIO() sys.stderr = self.stderr + store.register_opts(CONF) + self.stubs.Set(glance.common.config, 'load_paste_app', self._do_nothing) self.stubs.Set(glance.common.wsgi.Server, 'start', @@ -58,11 +65,11 @@ class TestGlanceApiCmd(test_utils.BaseTestCase): super(TestGlanceApiCmd, self).tearDown() def test_supported_default_store(self): - self.config(default_store='file') + self.config(group='glance_store', default_store='file') glance.cmd.api.main() def test_unsupported_default_store(self): - self.config(default_store='shouldnotexist') + self.config(group='glance_store', default_store='shouldnotexist') exit = self.assertRaises(SystemExit, glance.cmd.api.main) self.assertEqual(exit.code, 1) diff --git a/glance/tests/unit/base.py b/glance/tests/unit/base.py index d5c6554619..c3682511ba 100644 --- a/glance/tests/unit/base.py +++ b/glance/tests/unit/base.py @@ -17,20 +17,16 @@ import os import shutil import fixtures +import glance_store as store +from glance_store import location from oslo.config import cfg from oslo.db import options -from glance.common import exception from glance.openstack.common import jsonutils -from glance import store -from glance.store import location -from glance.store import sheepdog -from glance.store import vmware_datastore from glance.tests import stubs from glance.tests import utils as test_utils CONF = cfg.CONF -CONF.import_opt('filesystem_store_datadir', 'glance.store.filesystem') class StoreClearingUnitTest(test_utils.BaseTestCase): @@ -50,17 +46,8 @@ class StoreClearingUnitTest(test_utils.BaseTestCase): :param passing_config: making store driver passes basic configurations. :returns: the number of how many store drivers been loaded. """ - - def _fun(*args, **kwargs): - if passing_config: - return None - else: - raise exception.BadStoreConfiguration() - - self.stubs.Set(sheepdog.Store, 'configure', _fun) - self.stubs.Set(vmware_datastore.Store, 'configure', _fun) - self.stubs.Set(vmware_datastore.Store, 'configure_add', _fun) - return store.create_stores() + store.register_opts(CONF) + store.create_stores(CONF) class IsolatedUnitTest(StoreClearingUnitTest): @@ -77,12 +64,17 @@ class IsolatedUnitTest(StoreClearingUnitTest): policy_file = self._copy_data_file('policy.json', self.test_dir) options.set_defaults(CONF, connection='sqlite://', sqlite_db='glance.sqlite') + self.config(verbose=False, debug=False, - default_store='filesystem', - filesystem_store_datadir=os.path.join(self.test_dir), policy_file=policy_file, lock_path=os.path.join(self.test_dir)) + + self.config(default_store='filesystem', + filesystem_store_datadir=os.path.join(self.test_dir), + group="glance_store") + + store.create_stores() stubs.stub_out_registry_and_store_server(self.stubs, self.test_dir, registry=self.registry) diff --git a/glance/tests/unit/test_cinder_store.py b/glance/tests/unit/test_cinder_store.py deleted file mode 100644 index 9a82b7f43c..0000000000 --- a/glance/tests/unit/test_cinder_store.py +++ /dev/null @@ -1,84 +0,0 @@ -# Copyright 2013 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import stubout - -from cinderclient.v2 import client as cinderclient -import six - -from glance.common import exception -from glance.openstack.common import units -import glance.store.cinder as cinder -from glance.store.location import get_location_from_uri -from glance.tests.unit import base - - -class FakeObject(object): - def __init__(self, **kwargs): - for name, value in six.iteritems(kwargs): - setattr(self, name, value) - - -class TestCinderStore(base.StoreClearingUnitTest): - - def setUp(self): - self.config(default_store='cinder', - known_stores=['glance.store.cinder.Store']) - super(TestCinderStore, self).setUp() - self.stubs = stubout.StubOutForTesting() - - def test_cinder_configure_add(self): - store = cinder.Store() - self.assertRaises(exception.BadStoreConfiguration, - store.configure_add) - store = cinder.Store(context=None) - self.assertRaises(exception.BadStoreConfiguration, - store.configure_add) - store = cinder.Store(context=FakeObject(service_catalog=None)) - self.assertRaises(exception.BadStoreConfiguration, - store.configure_add) - store = cinder.Store(context=FakeObject( - service_catalog='fake_service_catalog')) - store.configure_add() - - def test_cinder_get_size(self): - fake_client = FakeObject(auth_token=None, management_url=None) - fake_volumes = {'12345678-9012-3455-6789-012345678901': - FakeObject(size=5)} - - class FakeCinderClient(FakeObject): - def __init__(self, *args, **kwargs): - super(FakeCinderClient, self).__init__(client=fake_client, - volumes=fake_volumes) - - self.stubs.Set(cinderclient, 'Client', FakeCinderClient) - - fake_sc = [{u'endpoints': [{u'publicURL': u'foo_public_url'}], - u'endpoints_links': [], - u'name': u'cinder', - u'type': u'volume'}] - fake_context = FakeObject(service_catalog=fake_sc, - user='fake_uer', - auth_tok='fake_token', - tenant='fake_tenant') - - uri = 'cinder://%s' % fake_volumes.keys()[0] - loc = get_location_from_uri(uri) - store = cinder.Store(context=fake_context) - image_size = store.get_size(loc) - self.assertEqual(image_size, - fake_volumes.values()[0].size * units.Gi) - self.assertEqual(fake_client.auth_token, 'fake_token') - self.assertEqual(fake_client.management_url, 'foo_public_url') diff --git a/glance/tests/unit/test_filesystem_store.py b/glance/tests/unit/test_filesystem_store.py deleted file mode 100644 index 487924cd13..0000000000 --- a/glance/tests/unit/test_filesystem_store.py +++ /dev/null @@ -1,462 +0,0 @@ -# Copyright 2011 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Tests the filesystem backend store""" - -import errno -import hashlib -import json -import os -import uuid - -import fixtures -from mock import patch -from oslo.config import cfg -import six -import six.moves.builtins as __builtin__ - -from glance.common import exception -from glance.openstack.common import units - -from glance.store.filesystem import Store -from glance.store.location import get_location_from_uri -from glance.tests.unit import base - -CONF = cfg.CONF - - -class TestStore(base.IsolatedUnitTest): - - def setUp(self): - """Establish a clean test environment""" - super(TestStore, self).setUp() - self.orig_read_chunksize = Store.READ_CHUNKSIZE - self.orig_write_chunksize = Store.WRITE_CHUNKSIZE - Store.READ_CHUNKSIZE = Store.WRITE_CHUNKSIZE = 10 - self.store = Store() - - def tearDown(self): - """Clear the test environment""" - super(TestStore, self).tearDown() - Store.READ_CHUNKSIZE = self.orig_read_chunksize - Store.WRITE_CHUNKSIZE = self.orig_write_chunksize - - def test_configure_add_single_datadir(self): - """ - Tests filesystem specified by filesystem_store_datadir - are parsed correctly. - """ - store = self.useFixture(fixtures.TempDir()).path - CONF.set_override('filesystem_store_datadir', store) - self.store.configure_add() - self.assertEqual(self.store.datadir, store) - - def test_configure_add_with_single_and_multi_datadirs(self): - """ - Tests BadStoreConfiguration exception is raised if both - filesystem_store_datadir and filesystem_store_datadirs are specified. - """ - store_map = [self.useFixture(fixtures.TempDir()).path, - self.useFixture(fixtures.TempDir()).path] - CONF.set_override('filesystem_store_datadirs', - [store_map[0] + ":100", - store_map[1] + ":200"]) - self.assertRaises(exception.BadStoreConfiguration, - self.store.configure_add) - - def test_configure_add_without_single_and_multi_datadirs(self): - """ - Tests BadStoreConfiguration exception is raised if neither - filesystem_store_datadir nor filesystem_store_datadirs are specified. - """ - CONF.clear_override('filesystem_store_datadir') - self.assertRaises(exception.BadStoreConfiguration, - self.store.configure_add) - - def test_configure_add_with_multi_datadirs(self): - """ - Tests multiple filesystem specified by filesystem_store_datadirs - are parsed correctly. - """ - store_map = [self.useFixture(fixtures.TempDir()).path, - self.useFixture(fixtures.TempDir()).path] - CONF.clear_override('filesystem_store_datadir') - CONF.set_override('filesystem_store_datadirs', - [store_map[0] + ":100", - store_map[1] + ":200"]) - self.store.configure_add() - - expected_priority_map = {100: [store_map[0]], 200: [store_map[1]]} - expected_priority_list = [200, 100] - self.assertEqual(self.store.priority_data_map, expected_priority_map) - self.assertEqual(self.store.priority_list, expected_priority_list) - - def test_configure_add_invalid_priority(self): - """ - Tests invalid priority specified by filesystem_store_datadirs - param raises BadStoreConfiguration exception. - """ - CONF.clear_override('filesystem_store_datadir') - CONF.set_override('filesystem_store_datadirs', - [self.useFixture(fixtures.TempDir()).path + ":100", - self.useFixture(fixtures.TempDir()).path + - ":invalid"]) - self.assertRaises(exception.BadStoreConfiguration, - self.store.configure_add) - - def test_configure_add_same_dir_multiple_times(self): - """ - Tests BadStoreConfiguration exception is raised if same directory - is specified multiple times in filesystem_store_datadirs. - """ - store_map = [self.useFixture(fixtures.TempDir()).path, - self.useFixture(fixtures.TempDir()).path] - CONF.clear_override('filesystem_store_datadir') - CONF.set_override('filesystem_store_datadirs', - [store_map[0] + ":100", - store_map[1] + ":200", - store_map[0] + ":300"]) - self.assertRaises(exception.BadStoreConfiguration, - self.store.configure_add) - - def test_configure_add_with_empty_datadir_path(self): - """ - Tests BadStoreConfiguration exception is raised if empty directory - path is specified in filesystem_store_datadirs. - """ - CONF.clear_override('filesystem_store_datadir') - CONF.set_override('filesystem_store_datadirs', ['']) - self.assertRaises(exception.BadStoreConfiguration, - self.store.configure_add) - - def test_configure_add_with_readonly_datadir_path(self): - """ - Tests BadStoreConfiguration exception is raised if directory - path specified in filesystem_store_datadirs is readonly. - """ - readonly_dir = self.useFixture(fixtures.TempDir()).path - os.chmod(readonly_dir, 0o444) - CONF.clear_override('filesystem_store_datadir') - CONF.set_override('filesystem_store_datadirs', [readonly_dir]) - self.assertRaises(exception.BadStoreConfiguration, - self.store.configure_add) - - def test_get(self): - """Test a "normal" retrieval of an image in chunks""" - # First add an image... - image_id = str(uuid.uuid4()) - file_contents = "chunk00000remainder" - image_file = six.StringIO(file_contents) - - location, size, checksum, _ = self.store.add(image_id, - image_file, - len(file_contents)) - - # Now read it back... - uri = "file:///%s/%s" % (self.test_dir, image_id) - loc = get_location_from_uri(uri) - (image_file, image_size) = self.store.get(loc) - - expected_data = "chunk00000remainder" - expected_num_chunks = 2 - data = "" - num_chunks = 0 - - for chunk in image_file: - num_chunks += 1 - data += chunk - self.assertEqual(expected_data, data) - self.assertEqual(expected_num_chunks, num_chunks) - - def test_get_non_existing(self): - """ - Test that trying to retrieve a file that doesn't exist - raises an error - """ - loc = get_location_from_uri("file:///%s/non-existing" % self.test_dir) - self.assertRaises(exception.NotFound, - self.store.get, - loc) - - def test_add(self): - """Test that we can add an image via the filesystem backend""" - Store.WRITE_CHUNKSIZE = 1024 - expected_image_id = str(uuid.uuid4()) - expected_file_size = 5 * units.Ki # 5K - expected_file_contents = "*" * expected_file_size - expected_checksum = hashlib.md5(expected_file_contents).hexdigest() - expected_location = "file://%s/%s" % (self.test_dir, - expected_image_id) - image_file = six.StringIO(expected_file_contents) - - location, size, checksum, _ = self.store.add(expected_image_id, - image_file, - expected_file_size) - - self.assertEqual(expected_location, location) - self.assertEqual(expected_file_size, size) - self.assertEqual(expected_checksum, checksum) - - uri = "file:///%s/%s" % (self.test_dir, expected_image_id) - loc = get_location_from_uri(uri) - (new_image_file, new_image_size) = self.store.get(loc) - new_image_contents = "" - new_image_file_size = 0 - - for chunk in new_image_file: - new_image_file_size += len(chunk) - new_image_contents += chunk - - self.assertEqual(expected_file_contents, new_image_contents) - self.assertEqual(expected_file_size, new_image_file_size) - - def test_add_with_multiple_dirs(self): - """Test adding multiple filesystem directories.""" - store_map = [self.useFixture(fixtures.TempDir()).path, - self.useFixture(fixtures.TempDir()).path] - CONF.clear_override('filesystem_store_datadir') - CONF.set_override('filesystem_store_datadirs', - [store_map[0] + ":100", - store_map[1] + ":200"]) - self.store.configure_add() - - """Test that we can add an image via the filesystem backend""" - Store.WRITE_CHUNKSIZE = 1024 - expected_image_id = str(uuid.uuid4()) - expected_file_size = 5 * units.Ki # 5K - expected_file_contents = "*" * expected_file_size - expected_checksum = hashlib.md5(expected_file_contents).hexdigest() - expected_location = "file://%s/%s" % (store_map[1], - expected_image_id) - image_file = six.StringIO(expected_file_contents) - - location, size, checksum, _ = self.store.add(expected_image_id, - image_file, - expected_file_size) - - self.assertEqual(expected_location, location) - self.assertEqual(expected_file_size, size) - self.assertEqual(expected_checksum, checksum) - - loc = get_location_from_uri(expected_location) - (new_image_file, new_image_size) = self.store.get(loc) - new_image_contents = "" - new_image_file_size = 0 - - for chunk in new_image_file: - new_image_file_size += len(chunk) - new_image_contents += chunk - - self.assertEqual(expected_file_contents, new_image_contents) - self.assertEqual(expected_file_size, new_image_file_size) - - def test_add_with_multiple_dirs_storage_full(self): - """ - Test StorageFull exception is raised if no filesystem directory - is found that can store an image. - """ - store_map = [self.useFixture(fixtures.TempDir()).path, - self.useFixture(fixtures.TempDir()).path] - CONF.clear_override('filesystem_store_datadir') - CONF.set_override('filesystem_store_datadirs', - [store_map[0] + ":100", - store_map[1] + ":200"]) - self.store.configure_add() - - def fake_get_capacity_info(mount_point): - return 0 - - self.stubs.Set(self.store, '_get_capacity_info', - fake_get_capacity_info) - Store.WRITE_CHUNKSIZE = 1024 - expected_image_id = str(uuid.uuid4()) - expected_file_size = 5 * units.Ki # 5K - expected_file_contents = "*" * expected_file_size - image_file = six.StringIO(expected_file_contents) - - self.assertRaises(exception.StorageFull, self.store.add, - expected_image_id, image_file, expected_file_size) - - def test_add_check_metadata_success(self): - expected_image_id = str(uuid.uuid4()) - in_metadata = {'akey': u'some value', 'list': [u'1', u'2', u'3']} - jsonfilename = os.path.join(self.test_dir, - "storage_metadata.%s" % expected_image_id) - - self.config(filesystem_store_metadata_file=jsonfilename) - with open(jsonfilename, 'w') as fptr: - json.dump(in_metadata, fptr) - expected_file_size = 10 - expected_file_contents = "*" * expected_file_size - image_file = six.StringIO(expected_file_contents) - - location, size, checksum, metadata = self.store.add(expected_image_id, - image_file, - expected_file_size) - - self.assertEqual(metadata, in_metadata) - - def test_add_check_metadata_bad_data(self): - expected_image_id = str(uuid.uuid4()) - in_metadata = {'akey': 10} # only unicode is allowed - jsonfilename = os.path.join(self.test_dir, - "storage_metadata.%s" % expected_image_id) - - self.config(filesystem_store_metadata_file=jsonfilename) - with open(jsonfilename, 'w') as fptr: - json.dump(in_metadata, fptr) - expected_file_size = 10 - expected_file_contents = "*" * expected_file_size - image_file = six.StringIO(expected_file_contents) - - location, size, checksum, metadata = self.store.add(expected_image_id, - image_file, - expected_file_size) - - self.assertEqual(metadata, {}) - - def test_add_check_metadata_bad_nosuch_file(self): - expected_image_id = str(uuid.uuid4()) - jsonfilename = os.path.join(self.test_dir, - "storage_metadata.%s" % expected_image_id) - - self.config(filesystem_store_metadata_file=jsonfilename) - expected_file_size = 10 - expected_file_contents = "*" * expected_file_size - image_file = six.StringIO(expected_file_contents) - - location, size, checksum, metadata = self.store.add(expected_image_id, - image_file, - expected_file_size) - - self.assertEqual(metadata, {}) - - def test_add_already_existing(self): - """ - Tests that adding an image with an existing identifier - raises an appropriate exception - """ - Store.WRITE_CHUNKSIZE = 1024 - image_id = str(uuid.uuid4()) - file_size = 5 * units.Ki # 5K - file_contents = "*" * file_size - image_file = six.StringIO(file_contents) - - location, size, checksum, _ = self.store.add(image_id, - image_file, - file_size) - image_file = six.StringIO("nevergonnamakeit") - self.assertRaises(exception.Duplicate, - self.store.add, - image_id, image_file, 0) - - def _do_test_add_write_failure(self, errno, exception): - Store.WRITE_CHUNKSIZE = 1024 - image_id = str(uuid.uuid4()) - file_size = 5 * units.Ki # 5K - file_contents = "*" * file_size - path = os.path.join(self.test_dir, image_id) - image_file = six.StringIO(file_contents) - - e = IOError() - e.errno = errno - with patch.object(__builtin__, 'open', side_effect=e) as mock_open: - self.assertRaises(exception, - self.store.add, - image_id, image_file, 0) - self.assertFalse(os.path.exists(path)) - mock_open.assert_called_once_with(path, 'wb') - - def test_add_storage_full(self): - """ - Tests that adding an image without enough space on disk - raises an appropriate exception - """ - self._do_test_add_write_failure(errno.ENOSPC, exception.StorageFull) - - def test_add_file_too_big(self): - """ - Tests that adding an excessively large image file - raises an appropriate exception - """ - self._do_test_add_write_failure(errno.EFBIG, exception.StorageFull) - - def test_add_storage_write_denied(self): - """ - Tests that adding an image with insufficient filestore permissions - raises an appropriate exception - """ - self._do_test_add_write_failure(errno.EACCES, - exception.StorageWriteDenied) - - def test_add_other_failure(self): - """ - Tests that a non-space-related IOError does not raise a - StorageFull exception. - """ - self._do_test_add_write_failure(errno.ENOTDIR, IOError) - - def test_add_cleanup_on_read_failure(self): - """ - Tests the partial image file is cleaned up after a read - failure. - """ - Store.WRITE_CHUNKSIZE = 1024 - image_id = str(uuid.uuid4()) - file_size = 5 * units.Ki # 5K - file_contents = "*" * file_size - path = os.path.join(self.test_dir, image_id) - image_file = six.StringIO(file_contents) - - def fake_Error(size): - raise AttributeError() - - self.stubs.Set(image_file, 'read', fake_Error) - - self.assertRaises(AttributeError, - self.store.add, - image_id, image_file, 0) - self.assertFalse(os.path.exists(path)) - - def test_delete(self): - """ - Test we can delete an existing image in the filesystem store - """ - # First add an image - image_id = str(uuid.uuid4()) - file_size = 5 * units.Ki # 5K - file_contents = "*" * file_size - image_file = six.StringIO(file_contents) - - location, size, checksum, _ = self.store.add(image_id, - image_file, - file_size) - - # Now check that we can delete it - uri = "file:///%s/%s" % (self.test_dir, image_id) - loc = get_location_from_uri(uri) - self.store.delete(loc) - - self.assertRaises(exception.NotFound, self.store.get, loc) - - def test_delete_non_existing(self): - """ - Test that trying to delete a file that doesn't exist - raises an error - """ - loc = get_location_from_uri("file:///tmp/glance-tests/non-existing") - self.assertRaises(exception.NotFound, - self.store.delete, - loc) diff --git a/glance/tests/unit/test_gridfs_store.py b/glance/tests/unit/test_gridfs_store.py deleted file mode 100644 index 843466d9ee..0000000000 --- a/glance/tests/unit/test_gridfs_store.py +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright 2013 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import six -import stubout - -from glance.common import exception -from glance.common import utils -from glance.store.gridfs import Store -from glance.tests.unit import base -try: - import gridfs - import pymongo -except ImportError: - pymongo = None - - -GRIDFS_CONF = {'verbose': True, - 'debug': True, - 'default_store': 'gridfs', - 'mongodb_store_uri': 'mongodb://fake_store_uri', - 'mongodb_store_db': 'fake_store_db'} - - -def stub_out_gridfs(stubs): - class FakeMongoClient(object): - def __init__(self, *args, **kwargs): - pass - - def __getitem__(self, key): - return None - - class FakeGridFS(object): - image_data = {} - called_commands = [] - - def __init__(self, *args, **kwargs): - pass - - def exists(self, image_id): - self.called_commands.append('exists') - return False - - def put(self, image_file, _id): - self.called_commands.append('put') - data = None - while True: - data = image_file.read(64) - if data: - self.image_data[_id] = \ - self.image_data.setdefault(_id, '') + data - else: - break - - def delete(self, _id): - self.called_commands.append('delete') - - if pymongo is not None: - stubs.Set(pymongo, 'MongoClient', FakeMongoClient) - stubs.Set(gridfs, 'GridFS', FakeGridFS) - - -class TestStore(base.StoreClearingUnitTest): - def setUp(self): - """Establish a clean test environment""" - self.config(**GRIDFS_CONF) - super(TestStore, self).setUp() - self.stubs = stubout.StubOutForTesting() - stub_out_gridfs(self.stubs) - self.store = Store() - self.addCleanup(self.stubs.UnsetAll) - - def test_cleanup_when_add_image_exception(self): - if pymongo is None: - msg = 'GridFS store can not add images, skip test.' - self.skipTest(msg) - - self.assertRaises(exception.ImageSizeLimitExceeded, - self.store.add, - 'fake_image_id', - utils.LimitingReader(six.StringIO('xx'), 1), - 2) - self.assertEqual(self.store.fs.called_commands, - ['exists', 'put', 'delete']) diff --git a/glance/tests/unit/test_http_store.py b/glance/tests/unit/test_http_store.py deleted file mode 100644 index ec53812ddc..0000000000 --- a/glance/tests/unit/test_http_store.py +++ /dev/null @@ -1,190 +0,0 @@ -# Copyright 2010-2011 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from six.moves import xrange -import stubout - -from glance.common import exception -from glance.common.store_utils import safe_delete_from_backend -from glance import context -from glance.db.sqlalchemy import api as db_api -from glance.registry.client.v1.api import configure_registry_client -from glance.store import delete_from_backend -from glance.store.http import MAX_REDIRECTS -from glance.store.http import Store -from glance.store.location import get_location_from_uri -from glance.tests import stubs as test_stubs -from glance.tests.unit import base -from glance.tests import utils - - -# The response stack is used to return designated responses in order; -# however when it's empty a default 200 OK response is returned from -# FakeHTTPConnection below. -FAKE_RESPONSE_STACK = [] - - -def stub_out_http_backend(stubs): - """ - Stubs out the httplib.HTTPRequest.getresponse to return - faked-out data instead of grabbing actual contents of a resource - - The stubbed getresponse() returns an iterator over - the data "I am a teapot, short and stout\n" - - :param stubs: Set of stubout stubs - """ - - class FakeHTTPConnection(object): - - def __init__(self, *args, **kwargs): - pass - - def getresponse(self): - if len(FAKE_RESPONSE_STACK): - return FAKE_RESPONSE_STACK.pop() - return utils.FakeHTTPResponse() - - def request(self, *_args, **_kwargs): - pass - - def close(self): - pass - - def fake_get_conn_class(self, *args, **kwargs): - return FakeHTTPConnection - - stubs.Set(Store, '_get_conn_class', fake_get_conn_class) - - -def stub_out_registry_image_update(stubs): - """ - Stubs an image update on the registry. - - :param stubs: Set of stubout stubs - """ - test_stubs.stub_out_registry_server(stubs) - - def fake_image_update(ctx, image_id, values, purge_props=False): - return {'properties': {}} - - stubs.Set(db_api, 'image_update', fake_image_update) - - -class TestHttpStore(base.StoreClearingUnitTest): - - def setUp(self): - global FAKE_RESPONSE_STACK - FAKE_RESPONSE_STACK = [] - self.config(default_store='http', - known_stores=['glance.store.http.Store']) - super(TestHttpStore, self).setUp() - self.stubs = stubout.StubOutForTesting() - stub_out_http_backend(self.stubs) - Store.READ_CHUNKSIZE = 2 - self.store = Store() - configure_registry_client() - - def test_http_get(self): - uri = "http://netloc/path/to/file.tar.gz" - expected_returns = ['I ', 'am', ' a', ' t', 'ea', 'po', 't,', ' s', - 'ho', 'rt', ' a', 'nd', ' s', 'to', 'ut', '\n'] - loc = get_location_from_uri(uri) - (image_file, image_size) = self.store.get(loc) - self.assertEqual(image_size, 31) - chunks = [c for c in image_file] - self.assertEqual(chunks, expected_returns) - - def test_http_get_redirect(self): - # Add two layers of redirects to the response stack, which will - # return the default 200 OK with the expected data after resolving - # both redirects. - redirect_headers_1 = {"location": "http://example.com/teapot.img"} - redirect_resp_1 = utils.FakeHTTPResponse(status=302, - headers=redirect_headers_1) - redirect_headers_2 = {"location": "http://example.com/teapot_real.img"} - redirect_resp_2 = utils.FakeHTTPResponse(status=301, - headers=redirect_headers_2) - FAKE_RESPONSE_STACK.append(redirect_resp_1) - FAKE_RESPONSE_STACK.append(redirect_resp_2) - - uri = "http://netloc/path/to/file.tar.gz" - expected_returns = ['I ', 'am', ' a', ' t', 'ea', 'po', 't,', ' s', - 'ho', 'rt', ' a', 'nd', ' s', 'to', 'ut', '\n'] - loc = get_location_from_uri(uri) - (image_file, image_size) = self.store.get(loc) - self.assertEqual(image_size, 31) - - chunks = [c for c in image_file] - self.assertEqual(chunks, expected_returns) - - def test_http_get_max_redirects(self): - # Add more than MAX_REDIRECTS redirects to the response stack - redirect_headers = {"location": "http://example.com/teapot.img"} - redirect_resp = utils.FakeHTTPResponse(status=302, - headers=redirect_headers) - for i in xrange(MAX_REDIRECTS + 2): - FAKE_RESPONSE_STACK.append(redirect_resp) - - uri = "http://netloc/path/to/file.tar.gz" - loc = get_location_from_uri(uri) - self.assertRaises(exception.MaxRedirectsExceeded, self.store.get, loc) - - def test_http_get_redirect_invalid(self): - redirect_headers = {"location": "http://example.com/teapot.img"} - redirect_resp = utils.FakeHTTPResponse(status=307, - headers=redirect_headers) - FAKE_RESPONSE_STACK.append(redirect_resp) - - uri = "http://netloc/path/to/file.tar.gz" - loc = get_location_from_uri(uri) - self.assertRaises(exception.BadStoreUri, self.store.get, loc) - - def test_http_get_not_found(self): - not_found_resp = utils.FakeHTTPResponse(status=404, - data="404 Not Found") - FAKE_RESPONSE_STACK.append(not_found_resp) - - uri = "http://netloc/path/to/file.tar.gz" - loc = get_location_from_uri(uri) - self.assertRaises(exception.NotFound, self.store.get, loc) - - def test_https_get(self): - uri = "https://netloc/path/to/file.tar.gz" - expected_returns = ['I ', 'am', ' a', ' t', 'ea', 'po', 't,', ' s', - 'ho', 'rt', ' a', 'nd', ' s', 'to', 'ut', '\n'] - loc = get_location_from_uri(uri) - (image_file, image_size) = self.store.get(loc) - self.assertEqual(image_size, 31) - - chunks = [c for c in image_file] - self.assertEqual(chunks, expected_returns) - - def test_http_delete_raise_error(self): - uri = "https://netloc/path/to/file.tar.gz" - loc = get_location_from_uri(uri) - ctx = context.RequestContext() - self.assertRaises(NotImplementedError, self.store.delete, loc) - self.assertRaises(exception.StoreDeleteNotSupported, - delete_from_backend, ctx, uri) - - def test_http_schedule_delete_swallows_error(self): - uri = {"url": "https://netloc/path/to/file.tar.gz"} - ctx = context.RequestContext() - stub_out_registry_image_update(self.stubs) - try: - safe_delete_from_backend(ctx, 'image_id', uri) - except exception.StoreDeleteNotSupported: - self.fail('StoreDeleteNotSupported should be swallowed') diff --git a/glance/tests/unit/test_image_cache.py b/glance/tests/unit/test_image_cache.py index 176569553f..cd7fbef4d9 100644 --- a/glance/tests/unit/test_image_cache.py +++ b/glance/tests/unit/test_image_cache.py @@ -17,7 +17,6 @@ from contextlib import contextmanager import datetime import hashlib import os -import tempfile import time import fixtures @@ -30,8 +29,6 @@ from glance import image_cache from glance.openstack.common import units #NOTE(bcwaldon): This is imported to load the registry config options import glance.registry # noqa -import glance.store.filesystem as fs_store -import glance.store.s3 as s3_store from glance.tests import utils as test_utils from glance.tests.utils import skip_if_disabled from glance.tests.utils import xattr_writes_supported @@ -409,44 +406,6 @@ class ImageCacheTestCase(object): # checksum is valid, fake image should be cached: self.assertTrue(cache.is_cached(image_id)) - def test_gate_caching_iter_fs_chunked_file(self): - """Tests get_caching_iter when using a filesystem ChunkedFile""" - image_id = 123 - - with tempfile.NamedTemporaryFile() as test_data_file: - test_data_file.write(FIXTURE_DATA) - test_data_file.seek(0) - image = fs_store.ChunkedFile(test_data_file.name) - md5 = hashlib.md5() - md5.update(FIXTURE_DATA) - checksum = md5.hexdigest() - - cache = image_cache.ImageCache() - img_iter = cache.get_caching_iter(image_id, checksum, image) - for chunk in img_iter: - pass - # checksum is valid, fake image should be cached: - self.assertTrue(cache.is_cached(image_id)) - - def test_gate_caching_iter_s3_chunked_file(self): - """Tests get_caching_iter when using an S3 ChunkedFile""" - image_id = 123 - - with tempfile.NamedTemporaryFile() as test_data_file: - test_data_file.write(FIXTURE_DATA) - test_data_file.seek(0) - image = s3_store.ChunkedFile(test_data_file) - md5 = hashlib.md5() - md5.update(FIXTURE_DATA) - checksum = md5.hexdigest() - - cache = image_cache.ImageCache() - img_iter = cache.get_caching_iter(image_id, checksum, image) - for chunk in img_iter: - pass - # checksum is valid, fake image should be cached: - self.assertTrue(cache.is_cached(image_id)) - def test_gate_caching_iter_bad_checksum(self): image = "12345678990abcdefghijklmnop" image_id = 123 diff --git a/glance/tests/unit/test_notifier.py b/glance/tests/unit/test_notifier.py index 45ca9f1885..78d5bc5fef 100644 --- a/glance/tests/unit/test_notifier.py +++ b/glance/tests/unit/test_notifier.py @@ -16,6 +16,7 @@ import datetime +import glance_store import mock from oslo.config import cfg from oslo import messaging @@ -254,7 +255,7 @@ class TestImageNotifications(utils.BaseTestCase): def data_iterator(): self.notifier.log = [] yield 'abcde' - raise exception.StorageFull('Modern Major General') + raise glance_store.StorageFull(message='Modern Major General') self.assertRaises(webob.exc.HTTPRequestEntityTooLarge, self.image_proxy.set_data, data_iterator(), 10) @@ -304,7 +305,7 @@ class TestImageNotifications(utils.BaseTestCase): def data_iterator(): self.notifier.log = [] yield 'abcde' - raise exception.StorageWriteDenied('The Very Model') + raise glance_store.StorageWriteDenied(message='The Very Model') self.assertRaises(webob.exc.HTTPServiceUnavailable, self.image_proxy.set_data, data_iterator(), 10) diff --git a/glance/tests/unit/test_rbd_store.py b/glance/tests/unit/test_rbd_store.py deleted file mode 100644 index b49cdff0c7..0000000000 --- a/glance/tests/unit/test_rbd_store.py +++ /dev/null @@ -1,187 +0,0 @@ -# Copyright 2013 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import mock -import six - -from glance.common import exception -from glance.common import utils -from glance.openstack.common import units -from glance.store.location import Location -import glance.store.rbd as rbd_store -from glance.store.rbd import StoreLocation -from glance.tests.unit import base -from glance.tests.unit.fake_rados import mock_rados -from glance.tests.unit.fake_rados import mock_rbd - - -class TestStore(base.StoreClearingUnitTest): - def setUp(self): - """Establish a clean test environment""" - super(TestStore, self).setUp() - self.stubs.Set(rbd_store, 'rados', mock_rados) - self.stubs.Set(rbd_store, 'rbd', mock_rbd) - self.store = rbd_store.Store() - self.store.chunk_size = 2 - self.called_commands_actual = [] - self.called_commands_expected = [] - self.store_specs = {'image': 'fake_image', - 'snapshot': 'fake_snapshot'} - self.location = StoreLocation(self.store_specs) - # Provide enough data to get more than one chunk iteration. - self.data_len = 3 * units.Ki - self.data_iter = six.StringIO('*' * self.data_len) - - def test_add_w_image_size_zero(self): - """Assert that correct size is returned even though 0 was provided.""" - self.store.chunk_size = units.Ki - with mock.patch.object(rbd_store.rbd.Image, 'resize') as resize: - with mock.patch.object(rbd_store.rbd.Image, 'write') as write: - ret = self.store.add('fake_image_id', self.data_iter, 0) - - resize.assert_called() - write.assert_called() - self.assertEqual(ret[1], self.data_len) - - def test_add_w_rbd_image_exception(self): - def _fake_create_image(*args, **kwargs): - self.called_commands_actual.append('create') - return self.location - - def _fake_delete_image(*args, **kwargs): - self.called_commands_actual.append('delete') - - def _fake_enter(*args, **kwargs): - raise exception.NotFound("") - - self.stubs.Set(self.store, '_create_image', _fake_create_image) - self.stubs.Set(self.store, '_delete_image', _fake_delete_image) - self.stubs.Set(mock_rbd.Image, '__enter__', _fake_enter) - - self.assertRaises(exception.NotFound, self.store.add, - 'fake_image_id', self.data_iter, self.data_len) - - self.called_commands_expected = ['create', 'delete'] - - def test_add_w_rbd_image_exception2(self): - def _fake_create_image(*args, **kwargs): - self.called_commands_actual.append('create') - return self.location - - def _fake_delete_image(*args, **kwargs): - self.called_commands_actual.append('delete') - raise exception.InUseByStore() - - def _fake_enter(*args, **kwargs): - raise exception.NotFound("") - - self.stubs.Set(self.store, '_create_image', _fake_create_image) - self.stubs.Set(self.store, '_delete_image', _fake_delete_image) - self.stubs.Set(mock_rbd.Image, '__enter__', _fake_enter) - - self.assertRaises(exception.InUseByStore, self.store.add, - 'fake_image_id', self.data_iter, self.data_len) - - self.called_commands_expected = ['create', 'delete'] - - def test_add_duplicate_image(self): - def _fake_create_image(*args, **kwargs): - self.called_commands_actual.append('create') - raise mock_rbd.ImageExists() - - self.stubs.Set(self.store, '_create_image', _fake_create_image) - self.assertRaises(exception.Duplicate, self.store.add, - 'fake_image_id', self.data_iter, self.data_len) - self.called_commands_expected = ['create'] - - def test_delete(self): - def _fake_remove(*args, **kwargs): - self.called_commands_actual.append('remove') - - self.stubs.Set(mock_rbd.RBD, 'remove', _fake_remove) - self.store.delete(Location('test_rbd_store', StoreLocation, - self.location.get_uri())) - self.called_commands_expected = ['remove'] - - def test__delete_image(self): - def _fake_remove(*args, **kwargs): - self.called_commands_actual.append('remove') - - self.stubs.Set(mock_rbd.RBD, 'remove', _fake_remove) - self.store._delete_image(self.location) - self.called_commands_expected = ['remove'] - - def test__delete_image_w_snap(self): - def _fake_unprotect_snap(*args, **kwargs): - self.called_commands_actual.append('unprotect_snap') - - def _fake_remove_snap(*args, **kwargs): - self.called_commands_actual.append('remove_snap') - - def _fake_remove(*args, **kwargs): - self.called_commands_actual.append('remove') - - self.stubs.Set(mock_rbd.RBD, 'remove', _fake_remove) - self.stubs.Set(mock_rbd.Image, 'unprotect_snap', _fake_unprotect_snap) - self.stubs.Set(mock_rbd.Image, 'remove_snap', _fake_remove_snap) - self.store._delete_image(self.location, snapshot_name='snap') - - self.called_commands_expected = ['unprotect_snap', 'remove_snap', - 'remove'] - - def test__delete_image_w_snap_exc_image_not_found(self): - def _fake_unprotect_snap(*args, **kwargs): - self.called_commands_actual.append('unprotect_snap') - raise mock_rbd.ImageNotFound() - - self.stubs.Set(mock_rbd.Image, 'unprotect_snap', _fake_unprotect_snap) - self.assertRaises(exception.NotFound, self.store._delete_image, - self.location, snapshot_name='snap') - - self.called_commands_expected = ['unprotect_snap'] - - def test__delete_image_exc_image_not_found(self): - def _fake_remove(*args, **kwargs): - self.called_commands_actual.append('remove') - raise mock_rbd.ImageNotFound() - - self.stubs.Set(mock_rbd.RBD, 'remove', _fake_remove) - self.assertRaises(exception.NotFound, self.store._delete_image, - self.location, snapshot_name='snap') - - self.called_commands_expected = ['remove'] - - def test_image_size_exceeded_exception(self): - def _fake_write(*args, **kwargs): - if 'write' not in self.called_commands_actual: - self.called_commands_actual.append('write') - raise exception.ImageSizeLimitExceeded - - def _fake_delete_image(*args, **kwargs): - self.called_commands_actual.append('delete') - - self.stubs.Set(mock_rbd.Image, 'write', _fake_write) - self.stubs.Set(self.store, '_delete_image', _fake_delete_image) - data = utils.LimitingReader(self.data_iter, self.data_len) - self.assertRaises(exception.ImageSizeLimitExceeded, - self.store.add, 'fake_image_id', - data, self.data_len + 1) - - self.called_commands_expected = ['write', 'delete'] - - def tearDown(self): - self.assertEqual(self.called_commands_actual, - self.called_commands_expected) - super(TestStore, self).tearDown() diff --git a/glance/tests/unit/test_s3_store.py b/glance/tests/unit/test_s3_store.py deleted file mode 100644 index 12ee394685..0000000000 --- a/glance/tests/unit/test_s3_store.py +++ /dev/null @@ -1,573 +0,0 @@ -# Copyright 2011 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Tests the S3 backend store""" - -import hashlib -import uuid -import xml.etree.ElementTree - -import boto.s3.connection -import mock -import six -import stubout - -from glance.common import exception -from glance.openstack.common import units - -from glance.store.location import get_location_from_uri -import glance.store.s3 -from glance.store.s3 import get_s3_location -from glance.store.s3 import Store -from glance.store import UnsupportedBackend -from glance.tests.unit import base - - -FAKE_UUID = str(uuid.uuid4()) - -FIVE_KB = 5 * units.Ki -S3_CONF = {'verbose': True, - 'debug': True, - 'default_store': 's3', - 's3_store_access_key': 'user', - 's3_store_secret_key': 'key', - 's3_store_host': 'localhost:8080', - 's3_store_bucket': 'glance', - 'known_stores': ['glance.store.s3.Store'], - 's3_store_large_object_size': 5, # over 5MB is large - 's3_store_large_object_chunk_size': 6} # part size is 6MB - -# ensure that mpu api is used and parts are uploaded as expected -mpu_parts_uploaded = 0 - - -# We stub out as little as possible to ensure that the code paths -# between glance.store.s3 and boto.s3.connection are tested -# thoroughly -def stub_out_s3(stubs): - - class FakeKey: - """ - Acts like a ``boto.s3.key.Key`` - """ - def __init__(self, bucket, name): - self.bucket = bucket - self.name = name - self.data = None - self.size = 0 - self.etag = None - self.BufferSize = 1024 - - def close(self): - pass - - def exists(self): - return self.bucket.exists(self.name) - - def delete(self): - self.bucket.delete(self.name) - - def compute_md5(self, data): - chunk = data.read(self.BufferSize) - checksum = hashlib.md5() - while chunk: - checksum.update(chunk) - chunk = data.read(self.BufferSize) - checksum_hex = checksum.hexdigest() - return checksum_hex, None - - def set_contents_from_file(self, fp, replace=False, **kwargs): - max_read = kwargs.get('size') - self.data = six.StringIO() - checksum = hashlib.md5() - while True: - if max_read is None or max_read > self.BufferSize: - read_size = self.BufferSize - elif max_read <= 0: - break - else: - read_size = max_read - chunk = fp.read(read_size) - if not chunk: - break - checksum.update(chunk) - self.data.write(chunk) - if max_read is not None: - max_read -= len(chunk) - self.size = self.data.len - # Reset the buffer to start - self.data.seek(0) - self.etag = checksum.hexdigest() - self.read = self.data.read - - def get_file(self): - return self.data - - class FakeMPU: - """ - Acts like a ``boto.s3.multipart.MultiPartUpload`` - """ - def __init__(self, bucket, key_name): - self.bucket = bucket - self.id = str(uuid.uuid4()) - self.key_name = key_name - self.parts = {} # pnum -> FakeKey - global mpu_parts_uploaded - mpu_parts_uploaded = 0 - - def upload_part_from_file(self, fp, part_num, **kwargs): - size = kwargs.get('size') - part = FakeKey(self.bucket, self.key_name) - part.set_contents_from_file(fp, size=size) - self.parts[part_num] = part - global mpu_parts_uploaded - mpu_parts_uploaded += 1 - return part - - def verify_xml(self, xml_body): - """ - Verify xml matches our part info. - """ - xmlparts = {} - cmuroot = xml.etree.ElementTree.fromstring(xml_body) - for cmupart in cmuroot: - pnum = int(cmupart.findtext('PartNumber')) - etag = cmupart.findtext('ETag') - xmlparts[pnum] = etag - if len(xmlparts) != len(self.parts): - return False - for pnum in xmlparts.keys(): - if self.parts[pnum] is None: - return False - if xmlparts[pnum] != self.parts[pnum].etag: - return False - return True - - def complete_key(self): - """ - Complete the parts into one big FakeKey - """ - key = FakeKey(self.bucket, self.key_name) - key.data = six.StringIO() - checksum = hashlib.md5() - cnt = 0 - for pnum in sorted(self.parts.keys()): - cnt += 1 - part = self.parts[pnum] - chunk = part.data.read(key.BufferSize) - while chunk: - checksum.update(chunk) - key.data.write(chunk) - chunk = part.data.read(key.BufferSize) - key.size = key.data.len - key.data.seek(0) - key.etag = checksum.hexdigest() + '-%d' % cnt - key.read = key.data.read - return key - - class FakeBucket: - """ - Acts like a ``boto.s3.bucket.Bucket`` - """ - def __init__(self, name, keys=None): - self.name = name - self.keys = keys or {} - self.mpus = {} # {key_name -> {id -> FakeMPU}} - - def __str__(self): - return self.name - - def exists(self, key): - return key in self.keys - - def delete(self, key): - del self.keys[key] - - def get_key(self, key_name, **kwargs): - return self.keys.get(key_name) - - def new_key(self, key_name): - new_key = FakeKey(self, key_name) - self.keys[key_name] = new_key - return new_key - - def initiate_multipart_upload(self, key_name, **kwargs): - mpu = FakeMPU(self, key_name) - if key_name not in self.mpus: - self.mpus[key_name] = {} - self.mpus[key_name][mpu.id] = mpu - return mpu - - def cancel_multipart_upload(self, key_name, upload_id, **kwargs): - if key_name in self.mpus: - if upload_id in self.mpus[key_name]: - del self.mpus[key_name][upload_id] - if not self.mpus[key_name]: - del self.mpus[key_name] - - def complete_multipart_upload(self, key_name, upload_id, - xml_body, **kwargs): - if key_name in self.mpus: - if upload_id in self.mpus[key_name]: - mpu = self.mpus[key_name][upload_id] - if mpu.verify_xml(xml_body): - key = mpu.complete_key() - self.cancel_multipart_upload(key_name, upload_id) - self.keys[key_name] = key - cmpu = mock.Mock() - cmpu.bucket = self - cmpu.bucket_name = self.name - cmpu.key_name = key_name - cmpu.etag = key.etag - return cmpu - return None # tho raising an exception might be better - - fixture_buckets = {'glance': FakeBucket('glance')} - b = fixture_buckets['glance'] - k = b.new_key(FAKE_UUID) - k.set_contents_from_file(six.StringIO("*" * FIVE_KB)) - - def fake_connection_constructor(self, *args, **kwargs): - host = kwargs.get('host') - if host.startswith('http://') or host.startswith('https://'): - raise UnsupportedBackend(host) - - def fake_get_bucket(conn, bucket_id): - bucket = fixture_buckets.get(bucket_id) - if not bucket: - bucket = FakeBucket(bucket_id) - return bucket - - stubs.Set(boto.s3.connection.S3Connection, - '__init__', fake_connection_constructor) - stubs.Set(boto.s3.connection.S3Connection, - 'get_bucket', fake_get_bucket) - - -def format_s3_location(user, key, authurl, bucket, obj): - """ - Helper method that returns a S3 store URI given - the component pieces. - """ - scheme = 's3' - if authurl.startswith('https://'): - scheme = 's3+https' - authurl = authurl[8:] - elif authurl.startswith('http://'): - authurl = authurl[7:] - authurl = authurl.strip('/') - return "%s://%s:%s@%s/%s/%s" % (scheme, user, key, authurl, - bucket, obj) - - -class TestStore(base.StoreClearingUnitTest): - - def setUp(self): - """Establish a clean test environment""" - self.config(**S3_CONF) - super(TestStore, self).setUp() - self.stubs = stubout.StubOutForTesting() - stub_out_s3(self.stubs) - self.store = Store() - self.addCleanup(self.stubs.UnsetAll) - - def test_get(self): - """Test a "normal" retrieval of an image in chunks""" - loc = get_location_from_uri( - "s3://user:key@auth_address/glance/%s" % FAKE_UUID) - (image_s3, image_size) = self.store.get(loc) - - self.assertEqual(image_size, FIVE_KB) - - expected_data = "*" * FIVE_KB - data = "" - - for chunk in image_s3: - data += chunk - self.assertEqual(expected_data, data) - - def test_get_calling_format_path(self): - """Test a "normal" retrieval of an image in chunks""" - self.config(s3_store_bucket_url_format='path') - - def fake_S3Connection_init(*args, **kwargs): - expected_cls = boto.s3.connection.OrdinaryCallingFormat - self.assertIsInstance(kwargs.get('calling_format'), expected_cls) - - self.stubs.Set(boto.s3.connection.S3Connection, '__init__', - fake_S3Connection_init) - - loc = get_location_from_uri( - "s3://user:key@auth_address/glance/%s" % FAKE_UUID) - (image_s3, image_size) = self.store.get(loc) - - def test_get_calling_format_default(self): - """Test a "normal" retrieval of an image in chunks""" - - def fake_S3Connection_init(*args, **kwargs): - expected_cls = boto.s3.connection.SubdomainCallingFormat - self.assertIsInstance(kwargs.get('calling_format'), expected_cls) - - self.stubs.Set(boto.s3.connection.S3Connection, '__init__', - fake_S3Connection_init) - - loc = get_location_from_uri( - "s3://user:key@auth_address/glance/%s" % FAKE_UUID) - (image_s3, image_size) = self.store.get(loc) - - def test_get_non_existing(self): - """ - Test that trying to retrieve a s3 that doesn't exist - raises an error - """ - uri = "s3://user:key@auth_address/badbucket/%s" % FAKE_UUID - loc = get_location_from_uri(uri) - self.assertRaises(exception.NotFound, self.store.get, loc) - - uri = "s3://user:key@auth_address/glance/noexist" - loc = get_location_from_uri(uri) - self.assertRaises(exception.NotFound, self.store.get, loc) - - def test_add(self): - """Test that we can add an image via the s3 backend""" - expected_image_id = str(uuid.uuid4()) - expected_s3_size = FIVE_KB - expected_s3_contents = "*" * expected_s3_size - expected_checksum = hashlib.md5(expected_s3_contents).hexdigest() - expected_location = format_s3_location( - S3_CONF['s3_store_access_key'], - S3_CONF['s3_store_secret_key'], - S3_CONF['s3_store_host'], - S3_CONF['s3_store_bucket'], - expected_image_id) - image_s3 = six.StringIO(expected_s3_contents) - - location, size, checksum, _ = self.store.add(expected_image_id, - image_s3, - expected_s3_size) - - self.assertEqual(expected_location, location) - self.assertEqual(expected_s3_size, size) - self.assertEqual(expected_checksum, checksum) - - loc = get_location_from_uri(expected_location) - (new_image_s3, new_image_size) = self.store.get(loc) - new_image_contents = six.StringIO() - for chunk in new_image_s3: - new_image_contents.write(chunk) - new_image_s3_size = new_image_contents.len - - self.assertEqual(expected_s3_contents, new_image_contents.getvalue()) - self.assertEqual(expected_s3_size, new_image_s3_size) - - def test_add_size_variations(self): - """ - Test that adding images of various sizes which exercise both S3 - single uploads and the multipart upload apis. We've configured - the big upload threshold to 5MB and the part size to 6MB. - """ - variations = [(FIVE_KB, 0), # simple put (5KB < 5MB) - (5242880, 1), # 1 part (5MB <= 5MB < 6MB) - (6291456, 1), # 1 part exact (5MB <= 6MB <= 6MB) - (7340032, 2)] # 2 parts (6MB < 7MB <= 12MB) - for (vsize, vcnt) in variations: - expected_image_id = str(uuid.uuid4()) - expected_s3_size = vsize - expected_s3_contents = "12345678" * (expected_s3_size / 8) - expected_chksum = hashlib.md5(expected_s3_contents).hexdigest() - expected_location = format_s3_location( - S3_CONF['s3_store_access_key'], - S3_CONF['s3_store_secret_key'], - S3_CONF['s3_store_host'], - S3_CONF['s3_store_bucket'], - expected_image_id) - image_s3 = six.StringIO(expected_s3_contents) - - # add image - location, size, chksum, _ = self.store.add(expected_image_id, - image_s3, - expected_s3_size) - self.assertEqual(expected_location, location) - self.assertEqual(expected_s3_size, size) - self.assertEqual(expected_chksum, chksum) - self.assertEqual(vcnt, mpu_parts_uploaded) - - # get image - loc = get_location_from_uri(expected_location) - (new_image_s3, new_image_s3_size) = self.store.get(loc) - new_image_contents = six.StringIO() - for chunk in new_image_s3: - new_image_contents.write(chunk) - new_image_size = new_image_contents.len - self.assertEqual(expected_s3_size, new_image_s3_size) - self.assertEqual(expected_s3_size, new_image_size) - self.assertEqual(expected_s3_contents, - new_image_contents.getvalue()) - - def test_add_host_variations(self): - """ - Test that having http(s):// in the s3serviceurl in config - options works as expected. - """ - variations = ['http://localhost:80', - 'http://localhost', - 'http://localhost/v1', - 'http://localhost/v1/', - 'https://localhost', - 'https://localhost:8080', - 'https://localhost/v1', - 'https://localhost/v1/', - 'localhost', - 'localhost:8080/v1'] - for variation in variations: - expected_image_id = str(uuid.uuid4()) - expected_s3_size = FIVE_KB - expected_s3_contents = "*" * expected_s3_size - expected_checksum = hashlib.md5(expected_s3_contents).hexdigest() - new_conf = S3_CONF.copy() - new_conf['s3_store_host'] = variation - expected_location = format_s3_location( - new_conf['s3_store_access_key'], - new_conf['s3_store_secret_key'], - new_conf['s3_store_host'], - new_conf['s3_store_bucket'], - expected_image_id) - image_s3 = six.StringIO(expected_s3_contents) - - self.config(**new_conf) - self.store = Store() - location, size, checksum, _ = self.store.add(expected_image_id, - image_s3, - expected_s3_size) - - self.assertEqual(expected_location, location) - self.assertEqual(expected_s3_size, size) - self.assertEqual(expected_checksum, checksum) - - loc = get_location_from_uri(expected_location) - (new_image_s3, new_image_size) = self.store.get(loc) - new_image_contents = new_image_s3.getvalue() - new_image_s3_size = len(new_image_s3) - - self.assertEqual(expected_s3_contents, new_image_contents) - self.assertEqual(expected_s3_size, new_image_s3_size) - - def test_add_already_existing(self): - """ - Tests that adding an image with an existing identifier - raises an appropriate exception - """ - image_s3 = six.StringIO("nevergonnamakeit") - self.assertRaises(exception.Duplicate, - self.store.add, - FAKE_UUID, image_s3, 0) - - def _option_required(self, key): - conf = S3_CONF.copy() - conf[key] = None - - try: - self.config(**conf) - self.store = Store() - return self.store.add == self.store.add_disabled - except Exception: - return False - return False - - def test_no_access_key(self): - """ - Tests that options without access key disables the add method - """ - self.assertTrue(self._option_required('s3_store_access_key')) - - def test_no_secret_key(self): - """ - Tests that options without secret key disables the add method - """ - self.assertTrue(self._option_required('s3_store_secret_key')) - - def test_no_host(self): - """ - Tests that options without host disables the add method - """ - self.assertTrue(self._option_required('s3_store_host')) - - def test_delete(self): - """ - Test we can delete an existing image in the s3 store - """ - uri = "s3://user:key@auth_address/glance/%s" % FAKE_UUID - loc = get_location_from_uri(uri) - self.store.delete(loc) - - self.assertRaises(exception.NotFound, self.store.get, loc) - - def test_delete_non_existing(self): - """ - Test that trying to delete a s3 that doesn't exist - raises an error - """ - uri = "s3://user:key@auth_address/glance/noexist" - loc = get_location_from_uri(uri) - self.assertRaises(exception.NotFound, self.store.delete, loc) - - def _do_test_get_s3_location(self, host, loc): - self.assertEqual(get_s3_location(host), loc) - self.assertEqual(get_s3_location(host + ':80'), loc) - self.assertEqual(get_s3_location('http://' + host), loc) - self.assertEqual(get_s3_location('http://' + host + ':80'), loc) - self.assertEqual(get_s3_location('https://' + host), loc) - self.assertEqual(get_s3_location('https://' + host + ':80'), loc) - - def test_get_s3_good_location(self): - """ - Test that the s3 location can be derived from the host - """ - good_locations = [ - ('s3.amazonaws.com', ''), - ('s3-eu-west-1.amazonaws.com', 'EU'), - ('s3-us-west-1.amazonaws.com', 'us-west-1'), - ('s3-ap-southeast-1.amazonaws.com', 'ap-southeast-1'), - ('s3-ap-northeast-1.amazonaws.com', 'ap-northeast-1'), - ] - for (url, expected) in good_locations: - self._do_test_get_s3_location(url, expected) - - def test_get_s3_bad_location(self): - """ - Test that the s3 location cannot be derived from an unexpected host - """ - bad_locations = [ - ('', ''), - ('s3.amazon.co.uk', ''), - ('s3-govcloud.amazonaws.com', ''), - ('cloudfiles.rackspace.com', ''), - ] - for (url, expected) in bad_locations: - self._do_test_get_s3_location(url, expected) - - def test_calling_format_path(self): - self.config(s3_store_bucket_url_format='path') - self.assertIsInstance(glance.store.s3.get_calling_format(), - boto.s3.connection.OrdinaryCallingFormat) - - def test_calling_format_subdomain(self): - self.config(s3_store_bucket_url_format='subdomain') - self.assertIsInstance(glance.store.s3.get_calling_format(), - boto.s3.connection.SubdomainCallingFormat) - - def test_calling_format_default(self): - self.assertIsInstance(glance.store.s3.get_calling_format(), - boto.s3.connection.SubdomainCallingFormat) diff --git a/glance/tests/unit/test_scrubber.py b/glance/tests/unit/test_scrubber.py index af7ed6fcfb..938559847f 100644 --- a/glance/tests/unit/test_scrubber.py +++ b/glance/tests/unit/test_scrubber.py @@ -19,22 +19,25 @@ import tempfile import uuid import eventlet +import glance_store import mox +from oslo.config import cfg from glance.common import exception - from glance import scrubber -import glance.store from glance.tests import utils as test_utils +CONF = cfg.CONF + class TestScrubber(test_utils.BaseTestCase): def setUp(self): self.data_dir = tempfile.mkdtemp() self.config(scrubber_datadir=self.data_dir) - self.config(default_store='file') - glance.store.create_stores() + glance_store.register_opts(CONF) + glance_store.create_stores() + self.config(group='glance_store', default_store='file') self.mox = mox.Mox() super(TestScrubber, self).setUp() @@ -49,12 +52,12 @@ class TestScrubber(test_utils.BaseTestCase): def _scrubber_cleanup_with_store_delete_exception(self, ex): uri = 'file://some/path/%s' % uuid.uuid4() id = 'helloworldid' - scrub = scrubber.Scrubber(glance.store) + scrub = scrubber.Scrubber(glance_store) scrub.registry = self.mox.CreateMockAnything() scrub.registry.get_image(id).AndReturn({'status': 'pending_delete'}) scrub.registry.update_image(id, {'status': 'deleted'}) - self.mox.StubOutWithMock(glance.store, "delete_from_backend") - glance.store.delete_from_backend( + self.mox.StubOutWithMock(glance_store, "delete_from_backend") + glance_store.delete_from_backend( mox.IgnoreArg(), uri).AndRaise(ex) self.mox.ReplayAll() @@ -66,7 +69,7 @@ class TestScrubber(test_utils.BaseTestCase): self.assertFalse(os.path.exists(q_path)) def test_store_delete_unsupported_backend_exception(self): - ex = glance.store.UnsupportedBackend() + ex = glance_store.UnsupportedBackend() self._scrubber_cleanup_with_store_delete_exception(ex) def test_store_delete_notfound_exception(self): diff --git a/glance/tests/unit/test_sheepdog_store.py b/glance/tests/unit/test_sheepdog_store.py deleted file mode 100644 index 7dc57ab2ce..0000000000 --- a/glance/tests/unit/test_sheepdog_store.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2013 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import six -import stubout - -from glance.common import exception -from glance.common import utils -from glance.openstack.common import processutils -import glance.store.sheepdog -from glance.store.sheepdog import Store -from glance.tests.unit import base - - -SHEEPDOG_CONF = {'verbose': True, - 'debug': True, - 'default_store': 'sheepdog'} - - -class TestStore(base.StoreClearingUnitTest): - def setUp(self): - """Establish a clean test environment""" - def _fake_execute(*cmd, **kwargs): - pass - - self.config(**SHEEPDOG_CONF) - super(TestStore, self).setUp() - self.stubs = stubout.StubOutForTesting() - self.stubs.Set(processutils, 'execute', _fake_execute) - self.store = Store() - self.addCleanup(self.stubs.UnsetAll) - - def test_cleanup_when_add_image_exception(self): - called_commands = [] - - def _fake_run_command(self, command, data, *params): - called_commands.append(command) - - self.stubs.Set(glance.store.sheepdog.SheepdogImage, - '_run_command', _fake_run_command) - - self.assertRaises(exception.ImageSizeLimitExceeded, - self.store.add, - 'fake_image_id', - utils.LimitingReader(six.StringIO('xx'), 1), - 2) - self.assertEqual([['list', '-r'], ['create'], ['delete']], - called_commands) diff --git a/glance/tests/unit/test_store_base.py b/glance/tests/unit/test_store_base.py deleted file mode 100644 index e0e0ecf171..0000000000 --- a/glance/tests/unit/test_store_base.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright 2011-2013 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from glance.common import exception -from glance.store import base as store_base -from glance.tests.unit import base as test_base - - -class FakeUnconfigurableStoreDriver(store_base.Store): - def configure(self): - raise exception.BadStoreConfiguration("Unconfigurable store driver.") - - -class TestStoreBase(test_base.StoreClearingUnitTest): - - class UnconfiguredStore(store_base.Store): - def add(self, image_id, image_file, image_size): - return True - - def delete(self, location): - return True - - def set_acls(self, location, public=False, read_tenants=None, - write_tenants=None): - return True - - def get_size(self, location): - return True - - def get(self, location): - return True - - def add_disabled(self, *args, **kwargs): - return True - - def setUp(self): - self.config(default_store='file') - super(TestStoreBase, self).setUp() - - def test_create_store_exclude_unconfigurable_drivers(self): - self.config(known_stores=[ - "glance.tests.unit.test_store_base.FakeUnconfigurableStoreDriver", - "glance.store.filesystem.Store"]) - count = self._create_stores(passing_config=True) - self.assertEqual(9, count) - count = self._create_stores(passing_config=False) - # Sheepdog and vshpere store driver - # needs to use default configure() - # to handle essential options. - self.assertEqual(7, count) - - def test_create_store_not_configured(self): - store = self.UnconfiguredStore(configure=False) - self.assertRaises(exception.StoreNotConfigured, store.add) - self.assertRaises(exception.StoreNotConfigured, store.get) - self.assertRaises(exception.StoreNotConfigured, store.get_size) - self.assertRaises(exception.StoreNotConfigured, store.add_disabled) - self.assertRaises(exception.StoreNotConfigured, store.delete) - self.assertRaises(exception.StoreNotConfigured, store.set_acls) - - def test_create_store_configured(self): - store = self.UnconfiguredStore(configure=True) - self.assertTrue(store.add) - self.assertTrue(store.get) - self.assertTrue(store.get_size) - self.assertTrue(store.add_disabled) - self.assertTrue(store.delete) - self.assertTrue(store.set_acls) diff --git a/glance/tests/unit/test_store_image.py b/glance/tests/unit/test_store_image.py index 032ca37c0a..232f05b693 100644 --- a/glance/tests/unit/test_store_image.py +++ b/glance/tests/unit/test_store_image.py @@ -14,9 +14,10 @@ # under the License. import mox +import glance_store + from glance.common import exception import glance.location -import glance.store from glance.tests.unit import utils as unit_test_utils from glance.tests import utils @@ -93,11 +94,11 @@ class TestStoreImage(utils.BaseTestCase): self.store_api, self.store_utils) location = image.locations[0] self.assertEqual(image.status, 'active') - self.store_api.get_from_backend({}, location['url']) + self.store_api.get_from_backend(location['url'], context={}) image.delete() self.assertEqual(image.status, 'deleted') - self.assertRaises(exception.NotFound, - self.store_api.get_from_backend, {}, location['url']) + self.assertRaises(glance_store.NotFound, + self.store_api.get_from_backend, location['url'], {}) def test_image_get_data(self): image = glance.location.ImageProxy(self.image_stub, {}, @@ -105,7 +106,7 @@ class TestStoreImage(utils.BaseTestCase): self.assertEqual(image.get_data(), 'XXX') def test_image_get_data_from_second_location(self): - def fake_get_from_backend(self, context, location): + def fake_get_from_backend(self, location, context=None): if UUID1 in location: raise Exception('not allow download from %s' % location) else: @@ -161,9 +162,9 @@ class TestStoreImage(utils.BaseTestCase): self.assertEqual(image.status, 'active') image.delete() self.assertEqual(image.status, 'deleted') - self.assertRaises(exception.NotFound, - self.store_api.get_from_backend, {}, - image.locations[0]['url']) + self.assertRaises(glance_store.NotFound, + self.store_api.get_from_backend, + image.locations[0]['url'], {}) def test_image_set_data_unknown_size(self): context = glance.context.RequestContext(user=USER1) @@ -178,9 +179,9 @@ class TestStoreImage(utils.BaseTestCase): self.assertEqual(image.status, 'active') image.delete() self.assertEqual(image.status, 'deleted') - self.assertRaises(exception.NotFound, - self.store_api.get_from_backend, {}, - image.locations[0]['url']) + self.assertRaises(glance_store.NotFound, + self.store_api.get_from_backend, + image.locations[0]['url'], context={}) def _add_image(self, context, image_id, data, len): image_stub = ImageStub(image_id, status='queued', locations=[]) @@ -225,7 +226,7 @@ class TestStoreImage(utils.BaseTestCase): # check below cases within 'TestStoreMetaDataChecker'. location_bad = {'url': UUID3, 'metadata': "a invalid metadata"} - self.assertRaises(glance.store.BackendException, + self.assertRaises(glance_store.BackendException, image1.locations.append, location_bad) image1.delete() @@ -314,7 +315,7 @@ class TestStoreImage(utils.BaseTestCase): location_bad = {'url': UUID3, 'metadata': "a invalid metadata"} - self.assertRaises(glance.store.BackendException, + self.assertRaises(glance_store.BackendException, image1.locations.extend, [location_bad]) image1.delete() @@ -416,7 +417,7 @@ class TestStoreImage(utils.BaseTestCase): location_bad = {'url': UUID3, 'metadata': "a invalid metadata"} - self.assertRaises(glance.store.BackendException, + self.assertRaises(glance_store.BackendException, image1.locations.insert, 0, location_bad) image1.delete() @@ -510,7 +511,7 @@ class TestStoreImage(utils.BaseTestCase): location_bad = {'url': UUID2, 'metadata': "a invalid metadata"} - self.assertRaises(glance.store.BackendException, + self.assertRaises(glance_store.BackendException, image2.locations.__iadd__, [location_bad]) self.assertEqual(image_stub2.locations, []) self.assertEqual(image2.locations, []) @@ -790,43 +791,43 @@ class TestImageFactory(utils.BaseTestCase): class TestStoreMetaDataChecker(utils.BaseTestCase): def test_empty(self): - glance.store.check_location_metadata({}) + glance_store.check_location_metadata({}) def test_unicode(self): m = {'key': u'somevalue'} - glance.store.check_location_metadata(m) + glance_store.check_location_metadata(m) def test_unicode_list(self): m = {'key': [u'somevalue', u'2']} - glance.store.check_location_metadata(m) + glance_store.check_location_metadata(m) def test_unicode_dict(self): inner = {'key1': u'somevalue', 'key2': u'somevalue'} m = {'topkey': inner} - glance.store.check_location_metadata(m) + glance_store.check_location_metadata(m) def test_unicode_dict_list(self): inner = {'key1': u'somevalue', 'key2': u'somevalue'} m = {'topkey': inner, 'list': [u'somevalue', u'2'], 'u': u'2'} - glance.store.check_location_metadata(m) + glance_store.check_location_metadata(m) def test_nested_dict(self): inner = {'key1': u'somevalue', 'key2': u'somevalue'} inner = {'newkey': inner} inner = {'anotherkey': inner} m = {'topkey': inner} - glance.store.check_location_metadata(m) + glance_store.check_location_metadata(m) def test_simple_bad(self): m = {'key1': object()} - self.assertRaises(glance.store.BackendException, - glance.store.check_location_metadata, + self.assertRaises(glance_store.BackendException, + glance_store.check_location_metadata, m) def test_list_bad(self): m = {'key1': [u'somevalue', object()]} - self.assertRaises(glance.store.BackendException, - glance.store.check_location_metadata, + self.assertRaises(glance_store.BackendException, + glance_store.check_location_metadata, m) def test_nested_dict_bad(self): @@ -835,8 +836,8 @@ class TestStoreMetaDataChecker(utils.BaseTestCase): inner = {'anotherkey': inner} m = {'topkey': inner} - self.assertRaises(glance.store.BackendException, - glance.store.check_location_metadata, + self.assertRaises(glance_store.BackendException, + glance_store.check_location_metadata, m) @@ -856,36 +857,36 @@ class TestStoreAddToBackend(utils.BaseTestCase): self.mox.UnsetStubs() def _bad_metadata(self, in_metadata): - store = self.mox.CreateMockAnything() - store.add(self.image_id, mox.IgnoreArg(), self.size).AndReturn( + mstore = self.mox.CreateMockAnything() + mstore.add(self.image_id, mox.IgnoreArg(), self.size).AndReturn( (self.location, self.size, self.checksum, in_metadata)) - store.__str__ = lambda: "hello" - store.__unicode__ = lambda: "hello" + mstore.__str__ = lambda: "hello" + mstore.__unicode__ = lambda: "hello" self.mox.ReplayAll() - self.assertRaises(glance.store.BackendException, - glance.store.store_add_to_backend, + self.assertRaises(glance_store.BackendException, + glance_store.store_add_to_backend, self.image_id, self.data, self.size, - store) + mstore) self.mox.VerifyAll() def _good_metadata(self, in_metadata): - store = self.mox.CreateMockAnything() - store.add(self.image_id, mox.IgnoreArg(), self.size).AndReturn( + mstore = self.mox.CreateMockAnything() + mstore.add(self.image_id, mox.IgnoreArg(), self.size).AndReturn( (self.location, self.size, self.checksum, in_metadata)) self.mox.ReplayAll() (location, size, checksum, - metadata) = glance.store.store_add_to_backend(self.image_id, + metadata) = glance_store.store_add_to_backend(self.image_id, self.data, self.size, - store) + mstore) self.mox.VerifyAll() self.assertEqual(self.location, location) self.assertEqual(self.size, size) @@ -940,8 +941,8 @@ class TestStoreAddToBackend(utils.BaseTestCase): self.mox.ReplayAll() - self.assertRaises(glance.store.BackendException, - glance.store.store_add_to_backend, + self.assertRaises(glance_store.BackendException, + glance_store.store_add_to_backend, self.image_id, self.data, self.size, diff --git a/glance/tests/unit/test_store_location.py b/glance/tests/unit/test_store_location.py index d483563c45..884221bd96 100644 --- a/glance/tests/unit/test_store_location.py +++ b/glance/tests/unit/test_store_location.py @@ -13,19 +13,11 @@ # License for the specific language governing permissions and limitations # under the License. -import fixtures import mock -from glance.common import exception -from glance import context +import glance_store + import glance.location -import glance.store -import glance.store.filesystem -import glance.store.http -import glance.store.location as location -import glance.store.s3 -import glance.store.swift -import glance.store.vmware_datastore from glance.tests.unit import base @@ -40,516 +32,18 @@ CONF = {'default_store': 'file', class TestStoreLocation(base.StoreClearingUnitTest): - def setUp(self): - self.config(default_store='file') - - # NOTE(flaper87): Each store should test - # this in their test suite. - self.config(known_stores=[ - "glance.store.filesystem.Store", - "glance.store.http.Store", - "glance.store.rbd.Store", - "glance.store.s3.Store", - "glance.store.swift.Store", - "glance.store.sheepdog.Store", - "glance.store.cinder.Store", - "glance.store.gridfs.Store", - "glance.store.vmware_datastore.Store", - ]) - conf = CONF.copy() - self.config(**conf) - reload(glance.store.swift) - super(TestStoreLocation, self).setUp() - - def test_get_location_from_uri_back_to_uri(self): - """ - Test that for various URIs, the correct Location - object can be constructed and then the original URI - returned via the get_store_uri() method. - """ - good_store_uris = [ - 'https://user:pass@example.com:80/images/some-id', - 'http://images.oracle.com/123456', - 'swift://account%3Auser:pass@authurl.com/container/obj-id', - 'swift://storeurl.com/container/obj-id', - 'swift+https://account%3Auser:pass@authurl.com/container/obj-id', - 's3://accesskey:secretkey@s3.amazonaws.com/bucket/key-id', - 's3://accesskey:secretwith/aslash@s3.amazonaws.com/bucket/key-id', - 's3+http://accesskey:secret@s3.amazonaws.com/bucket/key-id', - 's3+https://accesskey:secretkey@s3.amazonaws.com/bucket/key-id', - 'file:///var/lib/glance/images/1', - 'rbd://imagename', - 'rbd://fsid/pool/image/snap', - 'rbd://%2F/%2F/%2F/%2F', - 'sheepdog://244e75f1-9c69-4167-9db7-1aa7d1973f6c', - 'cinder://12345678-9012-3455-6789-012345678901', - 'vsphere://ip/folder/openstack_glance/2332298?dcPath=dc&dsName=ds', - ] - - for uri in good_store_uris: - loc = location.get_location_from_uri(uri) - # The get_store_uri() method *should* return an identical URI - # to the URI that is passed to get_location_from_uri() - self.assertEqual(loc.get_store_uri(), uri) - - def test_bad_store_scheme(self): - """ - Test that a URI with a non-existing scheme triggers exception - """ - bad_uri = 'unknown://user:pass@example.com:80/images/some-id' - - self.assertRaises(exception.UnknownScheme, - location.get_location_from_uri, - bad_uri) - - def test_filesystem_store_location(self): - """ - Test the specific StoreLocation for the Filesystem store - """ - uri = 'file:///var/lib/glance/images/1' - loc = glance.store.filesystem.StoreLocation({}) - loc.parse_uri(uri) - - self.assertEqual("file", loc.scheme) - self.assertEqual("/var/lib/glance/images/1", loc.path) - self.assertEqual(uri, loc.get_uri()) - - bad_uri = 'fil://' - self.assertRaises(AssertionError, loc.parse_uri, bad_uri) - - bad_uri = 'file://' - self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri) - - def test_http_store_location(self): - """ - Test the specific StoreLocation for the HTTP store - """ - uri = 'http://example.com/images/1' - loc = glance.store.http.StoreLocation({}) - loc.parse_uri(uri) - - self.assertEqual("http", loc.scheme) - self.assertEqual("example.com", loc.netloc) - self.assertEqual("/images/1", loc.path) - self.assertEqual(uri, loc.get_uri()) - - uri = 'https://example.com:8080/images/container/1' - loc.parse_uri(uri) - - self.assertEqual("https", loc.scheme) - self.assertEqual("example.com:8080", loc.netloc) - self.assertEqual("/images/container/1", loc.path) - self.assertEqual(uri, loc.get_uri()) - - uri = 'https://user:password@example.com:8080/images/container/1' - loc.parse_uri(uri) - - self.assertEqual("https", loc.scheme) - self.assertEqual("example.com:8080", loc.netloc) - self.assertEqual("user", loc.user) - self.assertEqual("password", loc.password) - self.assertEqual("/images/container/1", loc.path) - self.assertEqual(uri, loc.get_uri()) - - uri = 'https://user:@example.com:8080/images/1' - loc.parse_uri(uri) - - self.assertEqual("https", loc.scheme) - self.assertEqual("example.com:8080", loc.netloc) - self.assertEqual("user", loc.user) - self.assertEqual("", loc.password) - self.assertEqual("/images/1", loc.path) - self.assertEqual(uri, loc.get_uri()) - - bad_uri = 'htt://' - self.assertRaises(AssertionError, loc.parse_uri, bad_uri) - - bad_uri = 'http://' - self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri) - - bad_uri = 'http://user@example.com:8080/images/1' - self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri) - - def test_swift_store_location(self): - """ - Test the specific StoreLocation for the Swift store - """ - uri = 'swift+config://store_1/images/1' - loc = glance.store.swift.StoreLocation({}) - loc.parse_uri(uri) - - self.assertEqual("swift+config", loc.scheme) - self.assertEqual("localhost:8080", loc.auth_or_store_url) - self.assertEqual("https://localhost:8080", loc.swift_url) - self.assertEqual("images", loc.container) - self.assertEqual("1", loc.obj) - self.assertEqual('user', loc.user) - self.assertEqual('swift+https://user:key@localhost:8080/images/1', - loc.get_uri()) - - conf_file = "glance-swift.conf" - test_dir = self.useFixture(fixtures.TempDir()).path - self.swift_config_file = self._copy_data_file(conf_file, test_dir) - conf = CONF.copy() - conf.update({'swift_store_config_file': self.swift_config_file}) - self.config(**conf) - reload(glance.store.swift) - - uri = 'swift+config://store_2/images/1' - loc = glance.store.swift.StoreLocation({}) - loc.parse_uri(uri) - - self.assertEqual("swift+config", loc.scheme) - self.assertEqual("localhost:8080", loc.auth_or_store_url) - self.assertEqual("https://localhost:8080", loc.swift_url) - self.assertEqual("images", loc.container) - self.assertEqual("1", loc.obj) - self.assertEqual('tenant:user1', loc.user) - self.assertEqual('key1', loc.key) - self.assertEqual('swift+https://tenant%3Auser1:key1@localhost:8080' - '/images/1', - loc.get_uri()) - - uri = 'swift://example.com/images/1' - loc = glance.store.swift.StoreLocation({}) - loc.parse_uri(uri) - - self.assertEqual("swift", loc.scheme) - self.assertEqual("example.com", loc.auth_or_store_url) - self.assertEqual("https://example.com", loc.swift_url) - self.assertEqual("images", loc.container) - self.assertEqual("1", loc.obj) - self.assertIsNone(loc.user) - self.assertEqual(uri, loc.get_uri()) - - uri = 'swift+https://user:pass@authurl.com/images/1' - loc.parse_uri(uri) - - self.assertEqual("swift+https", loc.scheme) - self.assertEqual("authurl.com", loc.auth_or_store_url) - self.assertEqual("https://authurl.com", loc.swift_url) - self.assertEqual("images", loc.container) - self.assertEqual("1", loc.obj) - self.assertEqual("user", loc.user) - self.assertEqual("pass", loc.key) - self.assertEqual(uri, loc.get_uri()) - - uri = 'swift+https://user:pass@authurl.com/v1/container/12345' - loc.parse_uri(uri) - - self.assertEqual("swift+https", loc.scheme) - self.assertEqual("authurl.com/v1", loc.auth_or_store_url) - self.assertEqual("https://authurl.com/v1", loc.swift_url) - self.assertEqual("container", loc.container) - self.assertEqual("12345", loc.obj) - self.assertEqual("user", loc.user) - self.assertEqual("pass", loc.key) - self.assertEqual(uri, loc.get_uri()) - - uri = ('swift+http://a%3Auser%40example.com:p%40ss@authurl.com/' - 'v1/container/12345') - loc.parse_uri(uri) - - self.assertEqual("swift+http", loc.scheme) - self.assertEqual("authurl.com/v1", loc.auth_or_store_url) - self.assertEqual("http://authurl.com/v1", loc.swift_url) - self.assertEqual("container", loc.container) - self.assertEqual("12345", loc.obj) - self.assertEqual("a:user@example.com", loc.user) - self.assertEqual("p@ss", loc.key) - self.assertEqual(uri, loc.get_uri()) - - # multitenant puts store URL in the location (not auth) - uri = ('swift+http://storeurl.com/v1/container/12345') - loc.parse_uri(uri) - - self.assertEqual("swift+http", loc.scheme) - self.assertEqual("storeurl.com/v1", loc.auth_or_store_url) - self.assertEqual("http://storeurl.com/v1", loc.swift_url) - self.assertEqual("container", loc.container) - self.assertEqual("12345", loc.obj) - self.assertIsNone(loc.user) - self.assertIsNone(loc.key) - self.assertEqual(uri, loc.get_uri()) - - bad_uri = 'swif://' - self.assertRaises(AssertionError, loc.parse_uri, bad_uri) - - bad_uri = 'swift://' - self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri) - - bad_uri = 'swift://user@example.com:8080/images/1' - self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri) - - bad_uri = 'swift://user:pass@http://example.com:8080/images/1' - self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri) - - def test_s3_store_location(self): - """ - Test the specific StoreLocation for the S3 store - """ - uri = 's3://example.com/images/1' - loc = glance.store.s3.StoreLocation({}) - loc.parse_uri(uri) - - self.assertEqual("s3", loc.scheme) - self.assertEqual("example.com", loc.s3serviceurl) - self.assertEqual("images", loc.bucket) - self.assertEqual("1", loc.key) - self.assertIsNone(loc.accesskey) - self.assertEqual(uri, loc.get_uri()) - - uri = 's3+https://accesskey:pass@s3serviceurl.com/images/1' - loc.parse_uri(uri) - - self.assertEqual("s3+https", loc.scheme) - self.assertEqual("s3serviceurl.com", loc.s3serviceurl) - self.assertEqual("images", loc.bucket) - self.assertEqual("1", loc.key) - self.assertEqual("accesskey", loc.accesskey) - self.assertEqual("pass", loc.secretkey) - self.assertEqual(uri, loc.get_uri()) - - uri = 's3+https://accesskey:pass@s3serviceurl.com/v1/bucket/12345' - loc.parse_uri(uri) - - self.assertEqual("s3+https", loc.scheme) - self.assertEqual("s3serviceurl.com/v1", loc.s3serviceurl) - self.assertEqual("bucket", loc.bucket) - self.assertEqual("12345", loc.key) - self.assertEqual("accesskey", loc.accesskey) - self.assertEqual("pass", loc.secretkey) - self.assertEqual(uri, loc.get_uri()) - - uri = 's3://accesskey:pass/withslash@s3serviceurl.com/v1/bucket/12345' - loc.parse_uri(uri) - - self.assertEqual("s3", loc.scheme) - self.assertEqual("s3serviceurl.com/v1", loc.s3serviceurl) - self.assertEqual("bucket", loc.bucket) - self.assertEqual("12345", loc.key) - self.assertEqual("accesskey", loc.accesskey) - self.assertEqual("pass/withslash", loc.secretkey) - self.assertEqual(uri, loc.get_uri()) - - bad_uri = 's://' - self.assertRaises(AssertionError, loc.parse_uri, bad_uri) - - bad_uri = 's3://' - self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri) - - bad_uri = 's3://accesskey@example.com:8080/images/1' - self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri) - - bad_uri = 's3://user:pass@http://example.com:8080/images/1' - self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri) - - def test_rbd_store_location(self): - """ - Test the specific StoreLocation for the RBD store - """ - uri = 'rbd://imagename' - loc = glance.store.rbd.StoreLocation({}) - loc.parse_uri(uri) - - self.assertEqual('imagename', loc.image) - self.assertIsNone(loc.fsid) - self.assertIsNone(loc.pool) - self.assertIsNone(loc.snapshot) - - uri = u'rbd://imagename' - loc = glance.store.rbd.StoreLocation({}) - loc.parse_uri(uri) - - self.assertEqual('imagename', loc.image) - self.assertIsNone(loc.fsid) - self.assertIsNone(loc.pool) - self.assertIsNone(loc.snapshot) - - uri = 'rbd://fsid/pool/image/snap' - loc = glance.store.rbd.StoreLocation({}) - loc.parse_uri(uri) - - self.assertEqual('image', loc.image) - self.assertEqual('fsid', loc.fsid) - self.assertEqual('pool', loc.pool) - self.assertEqual('snap', loc.snapshot) - - uri = u'rbd://fsid/pool/image/snap' - loc = glance.store.rbd.StoreLocation({}) - loc.parse_uri(uri) - - self.assertEqual('image', loc.image) - self.assertEqual('fsid', loc.fsid) - self.assertEqual('pool', loc.pool) - self.assertEqual('snap', loc.snapshot) - - uri = 'rbd://%2f/%2f/%2f/%2f' - loc = glance.store.rbd.StoreLocation({}) - loc.parse_uri(uri) - - self.assertEqual('/', loc.image) - self.assertEqual('/', loc.fsid) - self.assertEqual('/', loc.pool) - self.assertEqual('/', loc.snapshot) - - uri = u'rbd://%2f/%2f/%2f/%2f' - loc = glance.store.rbd.StoreLocation({}) - loc.parse_uri(uri) - - self.assertEqual('/', loc.image) - self.assertEqual('/', loc.fsid) - self.assertEqual('/', loc.pool) - self.assertEqual('/', loc.snapshot) - - bad_uri = 'rbd:/image' - self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri) - - bad_uri = 'rbd://image/extra' - self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri) - - bad_uri = 'rbd://image/' - self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri) - - bad_uri = 'http://image' - self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri) - - bad_uri = 'http://fsid/pool/image/snap' - self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri) - - bad_uri = 'rbd://fsid/pool/image/' - self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri) - - bad_uri = 'rbd://fsid/pool/image/snap/' - self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri) - - bad_uri = 'http://///' - self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri) - - bad_uri = 'rbd://' + unichr(300) - self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri) - - def test_sheepdog_store_location(self): - """ - Test the specific StoreLocation for the Sheepdog store - """ - uri = 'sheepdog://244e75f1-9c69-4167-9db7-1aa7d1973f6c' - loc = glance.store.sheepdog.StoreLocation({}) - loc.parse_uri(uri) - self.assertEqual('244e75f1-9c69-4167-9db7-1aa7d1973f6c', loc.image) - - bad_uri = 'sheepdog:/244e75f1-9c69-4167-9db7-1aa7d1973f6c' - self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri) - - bad_uri = 'http://244e75f1-9c69-4167-9db7-1aa7d1973f6c' - self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri) - - bad_uri = 'image; name' - self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri) - - def test_vmware_store_location(self): - """ - Test the specific StoreLocation for the VMware store - """ - ds_url_prefix = glance.store.vmware_datastore.DS_URL_PREFIX - image_dir = glance.store.vmware_datastore.DEFAULT_STORE_IMAGE_DIR - uri = ('vsphere://127.0.0.1%s%s/29038321?dcPath=my-dc&dsName=my-ds' % - (ds_url_prefix, image_dir)) - loc = glance.store.vmware_datastore.StoreLocation({}) - loc.parse_uri(uri) - - self.assertEqual("vsphere", loc.scheme) - self.assertEqual("127.0.0.1", loc.server_host) - self.assertEqual("%s%s/29038321" % - (ds_url_prefix, image_dir), loc.path) - self.assertEqual("dcPath=my-dc&dsName=my-ds", loc.query) - self.assertEqual(uri, loc.get_uri()) - - bad_uri = 'vphere://' - self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri) - - bad_uri = ('vspheer://127.0.0.1%s%s/29038321?dcPath=my-dc&dsName=my-ds' - % (ds_url_prefix, image_dir)) - self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri) - - bad_uri = ('http://127.0.0.1%s%s/29038321?dcPath=my-dc&dsName=my-ds' - % (ds_url_prefix, image_dir)) - self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri) - - bad_uri = ('vsphere:/127.0.0.1%s%s/29038321?dcPath=my-dc&dsName=my-ds' - % (ds_url_prefix, image_dir)) - self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri) - - bad_uri = ('vsphere://127.0.0.1%s%s/29038321?dcPath=my-dc&dsName=my-ds' - % (ds_url_prefix, "/folder_not_in_configuration")) - self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri) - - bad_uri = ('vsphere://127.0.0.1%s%s/29038321?dcPath=my-dc&dsName=my-ds' - % ("/wrong_folder_path", image_dir)) - self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri) - - def test_cinder_store_good_location(self): - """ - Test the specific StoreLocation for the Cinder store - """ - good_uri = 'cinder://12345678-9012-3455-6789-012345678901' - loc = glance.store.cinder.StoreLocation({}) - loc.parse_uri(good_uri) - self.assertEqual('12345678-9012-3455-6789-012345678901', loc.volume_id) - - def test_cinder_store_bad_location(self): - """ - Test the specific StoreLocation for the Cinder store - """ - bad_uri = 'cinder://volume-id-is-a-uuid' - loc = glance.store.cinder.StoreLocation({}) - self.assertRaises(exception.BadStoreUri, loc.parse_uri, bad_uri) - - def test_get_store_from_scheme(self): - """ - Test that the backend returned by glance.store.get_backend_class - is correct or raises an appropriate error. - """ - good_results = { - 'swift': glance.store.swift.SingleTenantStore, - 'swift+http': glance.store.swift.SingleTenantStore, - 'swift+https': glance.store.swift.SingleTenantStore, - 's3': glance.store.s3.Store, - 's3+http': glance.store.s3.Store, - 's3+https': glance.store.s3.Store, - 'file': glance.store.filesystem.Store, - 'filesystem': glance.store.filesystem.Store, - 'http': glance.store.http.Store, - 'https': glance.store.http.Store, - 'rbd': glance.store.rbd.Store, - 'sheepdog': glance.store.sheepdog.Store, - 'cinder': glance.store.cinder.Store, - 'vsphere': glance.store.vmware_datastore.Store} - - ctx = context.RequestContext() - for scheme, store in good_results.items(): - store_obj = glance.store.get_store_from_scheme(ctx, scheme) - self.assertEqual(store_obj.__class__, store) - - bad_results = ['fil', 'swift+h', 'unknown'] - - for store in bad_results: - self.assertRaises(exception.UnknownScheme, - glance.store.get_store_from_scheme, - ctx, - store) - def test_add_location_for_image_without_size(self): class FakeImageProxy(): size = None context = None store_api = mock.Mock() - def fake_get_size_from_backend(context, uri): + def fake_get_size_from_backend(uri, context=None): return 1 - self.stubs.Set(glance.store, 'get_size_from_backend', + self.stubs.Set(glance_store, 'get_size_from_backend', fake_get_size_from_backend) + with mock.patch('glance.location._check_image_location'): loc1 = {'url': 'file:///fake1.img.tar.gz', 'metadata': {}} loc2 = {'url': 'file:///fake2.img.tar.gz', 'metadata': {}} diff --git a/glance/tests/unit/test_swift_store.py b/glance/tests/unit/test_swift_store.py deleted file mode 100644 index 696b6f3d7c..0000000000 --- a/glance/tests/unit/test_swift_store.py +++ /dev/null @@ -1,1126 +0,0 @@ -# Copyright 2011 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Tests the Swift backend store""" - -import copy -import fixtures -import hashlib -import httplib -import mock -import tempfile -import uuid - -from oslo.config import cfg -import six -import stubout -import swiftclient - -import glance.common.auth -from glance.common import exception -from glance.common import swift_store_utils -from glance.common import utils -from glance.openstack.common import units - -from glance.store import BackendException -from glance.store.location import get_location_from_uri -from glance.store import swift -from glance.store.swift import swift_retry_iter -from glance.tests.unit import base - - -CONF = cfg.CONF - -FAKE_UUID = lambda: str(uuid.uuid4()) - -Store = glance.store.swift.Store -FIVE_KB = 5 * units.Ki -FIVE_GB = 5 * units.Gi -MAX_SWIFT_OBJECT_SIZE = FIVE_GB -SWIFT_PUT_OBJECT_CALLS = 0 -SWIFT_CONF = {'verbose': True, - 'debug': True, - 'known_stores': ['glance.store.swift.Store'], - 'default_store': 'swift', - 'swift_store_auth_address': 'localhost:8080', - 'swift_store_container': 'glance', - 'swift_store_user': 'user', - 'swift_store_key': 'key', - 'swift_store_auth_address': 'localhost:8080', - 'swift_store_container': 'glance', - 'swift_store_retry_get_count': 1, - 'default_swift_reference': 'ref1' - } - - -# We stub out as little as possible to ensure that the code paths -# between glance.store.swift and swiftclient are tested -# thoroughly -def stub_out_swiftclient(stubs, swift_store_auth_version): - fixture_containers = ['glance'] - fixture_container_headers = {} - fixture_headers = { - 'glance/%s' % FAKE_UUID: { - 'content-length': FIVE_KB, - 'etag': 'c2e5db72bd7fd153f53ede5da5a06de3' - } - } - fixture_objects = {'glance/%s' % FAKE_UUID: - six.StringIO("*" * FIVE_KB)} - - def fake_head_container(url, token, container, **kwargs): - if container not in fixture_containers: - msg = "No container %s found" % container - raise swiftclient.ClientException(msg, - http_status=httplib.NOT_FOUND) - return fixture_container_headers - - def fake_put_container(url, token, container, **kwargs): - fixture_containers.append(container) - - def fake_post_container(url, token, container, headers, http_conn=None): - for key, value in six.iteritems(headers): - fixture_container_headers[key] = value - - def fake_put_object(url, token, container, name, contents, **kwargs): - # PUT returns the ETag header for the newly-added object - # Large object manifest... - global SWIFT_PUT_OBJECT_CALLS - SWIFT_PUT_OBJECT_CALLS += 1 - CHUNKSIZE = swift.BaseStore.READ_CHUNKSIZE - fixture_key = "%s/%s" % (container, name) - if fixture_key not in fixture_headers: - if kwargs.get('headers'): - etag = kwargs['headers']['ETag'] - fixture_headers[fixture_key] = {'manifest': True, - 'etag': etag} - return etag - if hasattr(contents, 'read'): - fixture_object = six.StringIO() - chunk = contents.read(CHUNKSIZE) - checksum = hashlib.md5() - while chunk: - fixture_object.write(chunk) - checksum.update(chunk) - chunk = contents.read(CHUNKSIZE) - etag = checksum.hexdigest() - else: - fixture_object = six.StringIO(contents) - etag = hashlib.md5(fixture_object.getvalue()).hexdigest() - read_len = fixture_object.len - if read_len > MAX_SWIFT_OBJECT_SIZE: - msg = ('Image size:%d exceeds Swift max:%d' % - (read_len, MAX_SWIFT_OBJECT_SIZE)) - raise swiftclient.ClientException( - msg, http_status=httplib.REQUEST_ENTITY_TOO_LARGE) - fixture_objects[fixture_key] = fixture_object - fixture_headers[fixture_key] = { - 'content-length': read_len, - 'etag': etag} - return etag - else: - msg = ("Object PUT failed - Object with key %s already exists" - % fixture_key) - raise swiftclient.ClientException(msg, - http_status=httplib.CONFLICT) - - def fake_get_object(url, token, container, name, **kwargs): - # GET returns the tuple (list of headers, file object) - fixture_key = "%s/%s" % (container, name) - if fixture_key not in fixture_headers: - msg = "Object GET failed" - raise swiftclient.ClientException(msg, - http_status=httplib.NOT_FOUND) - - byte_range = None - headers = kwargs.get('headers', dict()) - if headers is not None: - headers = dict((k.lower(), v) for k, v in six.iteritems(headers)) - if 'range' in headers: - byte_range = headers.get('range') - - fixture = fixture_headers[fixture_key] - if 'manifest' in fixture: - # Large object manifest... we return a file containing - # all objects with prefix of this fixture key - chunk_keys = sorted([k for k in fixture_headers.keys() - if k.startswith(fixture_key) and - k != fixture_key]) - result = six.StringIO() - for key in chunk_keys: - result.write(fixture_objects[key].getvalue()) - else: - result = fixture_objects[fixture_key] - - if byte_range is not None: - start = int(byte_range.split('=')[1].strip('-')) - result = six.StringIO(result.getvalue()[start:]) - fixture_headers[fixture_key]['content-length'] = len( - result.getvalue()) - - return fixture_headers[fixture_key], result - - def fake_head_object(url, token, container, name, **kwargs): - # HEAD returns the list of headers for an object - try: - fixture_key = "%s/%s" % (container, name) - return fixture_headers[fixture_key] - except KeyError: - msg = "Object HEAD failed - Object does not exist" - raise swiftclient.ClientException(msg, - http_status=httplib.NOT_FOUND) - - def fake_delete_object(url, token, container, name, **kwargs): - # DELETE returns nothing - fixture_key = "%s/%s" % (container, name) - if fixture_key not in fixture_headers: - msg = "Object DELETE failed - Object does not exist" - raise swiftclient.ClientException(msg, - http_status=httplib.NOT_FOUND) - else: - del fixture_headers[fixture_key] - del fixture_objects[fixture_key] - - def fake_http_connection(*args, **kwargs): - return None - - def fake_get_auth(url, user, key, snet, auth_version, **kwargs): - if url is None: - return None, None - if 'http' in url and '://' not in url: - raise ValueError('Invalid url %s' % url) - # Check the auth version against the configured value - if swift_store_auth_version != auth_version: - msg = 'AUTHENTICATION failed (version mismatch)' - raise swiftclient.ClientException(msg) - return None, None - - stubs.Set(swiftclient.client, - 'head_container', fake_head_container) - stubs.Set(swiftclient.client, - 'put_container', fake_put_container) - stubs.Set(swiftclient.client, - 'post_container', fake_post_container) - stubs.Set(swiftclient.client, - 'put_object', fake_put_object) - stubs.Set(swiftclient.client, - 'delete_object', fake_delete_object) - stubs.Set(swiftclient.client, - 'head_object', fake_head_object) - stubs.Set(swiftclient.client, - 'get_object', fake_get_object) - stubs.Set(swiftclient.client, - 'get_auth', fake_get_auth) - stubs.Set(swiftclient.client, - 'http_connection', fake_http_connection) - - -class SwiftTests(object): - - @property - def swift_store_user(self): - return 'tenant:user1' - - def test_get_size(self): - """ - Test that we can get the size of an object in the swift store - """ - uri = "swift://%s:key@auth_address/glance/%s" % ( - self.swift_store_user, FAKE_UUID) - loc = get_location_from_uri(uri) - image_size = self.store.get_size(loc) - self.assertEqual(image_size, 5120) - - def test_validate_location_for_invalid_uri(self): - """ - Test that validate location raises when the location contains - any account reference. - """ - uri = "swift+config://store_1/glance/%s" - self.assertRaises(exception.BadStoreUri, - self.store.validate_location, - uri) - - def test_validate_location_for_valid_uri(self): - """ - Test that validate location verifies that the location does not - contain any account reference - """ - uri = "swift://user:key@auth_address/glance/%s" - try: - self.assertIsNone(self.store.validate_location(uri)) - except Exception: - self.fail('Location uri validation failed') - - def test_get_size_with_multi_tenant_on(self): - """Test that single tenant uris work with multi tenant on.""" - uri = ("swift://%s:key@auth_address/glance/%s" % - (self.swift_store_user, FAKE_UUID)) - self.config(swift_store_multi_tenant=True) - #NOTE(markwash): ensure the image is found - context = glance.context.RequestContext() - size = glance.store.get_size_from_backend(context, uri) - self.assertEqual(size, 5120) - - def test_get(self): - """Test a "normal" retrieval of an image in chunks""" - uri = "swift://%s:key@auth_address/glance/%s" % ( - self.swift_store_user, FAKE_UUID) - loc = get_location_from_uri(uri) - (image_swift, image_size) = self.store.get(loc) - self.assertEqual(image_size, 5120) - - expected_data = "*" * FIVE_KB - data = "" - - for chunk in image_swift: - data += chunk - self.assertEqual(expected_data, data) - - def test_get_with_retry(self): - """ - Test a retrieval where Swift does not get the full image in a single - request. - """ - uri = "swift://%s:key@auth_address/glance/%s" % ( - self.swift_store_user, FAKE_UUID) - loc = get_location_from_uri(uri) - (image_swift, image_size) = self.store.get(loc) - resp_full = ''.join([chunk for chunk in image_swift.wrapped]) - resp_half = resp_full[:len(resp_full) / 2] - image_swift.wrapped = swift_retry_iter(resp_half, image_size, - self.store, - loc.store_location) - self.assertEqual(image_size, 5120) - - expected_data = "*" * FIVE_KB - data = "" - - for chunk in image_swift: - data += chunk - self.assertEqual(expected_data, data) - - def test_get_with_http_auth(self): - """ - Test a retrieval from Swift with an HTTP authurl. This is - specified either via a Location header with swift+http:// or using - http:// in the swift_store_auth_address config value - """ - loc = get_location_from_uri("swift+http://%s:key@auth_address/" - "glance/%s" % - (self.swift_store_user, FAKE_UUID)) - (image_swift, image_size) = self.store.get(loc) - self.assertEqual(image_size, 5120) - - expected_data = "*" * FIVE_KB - data = "" - - for chunk in image_swift: - data += chunk - self.assertEqual(expected_data, data) - - def test_get_non_existing(self): - """ - Test that trying to retrieve a swift that doesn't exist - raises an error - """ - loc = get_location_from_uri("swift://%s:key@authurl/glance/noexist" % ( - self.swift_store_user)) - self.assertRaises(exception.NotFound, - self.store.get, - loc) - - def test_add(self): - """Test that we can add an image via the swift backend""" - swift_store_utils.is_multiple_swift_store_accounts_enabled = \ - mock.Mock(return_value=False) - reload(swift) - self.store = Store() - expected_swift_size = FIVE_KB - expected_swift_contents = "*" * expected_swift_size - expected_checksum = hashlib.md5(expected_swift_contents).hexdigest() - expected_image_id = str(uuid.uuid4()) - loc = "swift+https://tenant%%3Auser1:key@localhost:8080/glance/%s" - expected_location = loc % (expected_image_id) - image_swift = six.StringIO(expected_swift_contents) - - global SWIFT_PUT_OBJECT_CALLS - SWIFT_PUT_OBJECT_CALLS = 0 - - location, size, checksum, _ = self.store.add(expected_image_id, - image_swift, - expected_swift_size) - - self.assertEqual(expected_location, location) - self.assertEqual(expected_swift_size, size) - self.assertEqual(expected_checksum, checksum) - # Expecting a single object to be created on Swift i.e. no chunking. - self.assertEqual(SWIFT_PUT_OBJECT_CALLS, 1) - - loc = get_location_from_uri(expected_location) - (new_image_swift, new_image_size) = self.store.get(loc) - new_image_contents = ''.join([chunk for chunk in new_image_swift]) - new_image_swift_size = len(new_image_swift) - - self.assertEqual(expected_swift_contents, new_image_contents) - self.assertEqual(expected_swift_size, new_image_swift_size) - - def test_add_multi_store(self): - - conf = copy.deepcopy(SWIFT_CONF) - conf['default_swift_reference'] = 'store_2' - self.config(**conf) - reload(swift) - self.store = Store() - - expected_swift_size = FIVE_KB - expected_swift_contents = "*" * expected_swift_size - expected_image_id = str(uuid.uuid4()) - image_swift = six.StringIO(expected_swift_contents) - global SWIFT_PUT_OBJECT_CALLS - SWIFT_PUT_OBJECT_CALLS = 0 - loc = 'swift+config://store_2/glance/%s' - - expected_location = loc % (expected_image_id) - - location, size, checksum, arg = self.store.add(expected_image_id, - image_swift, - expected_swift_size) - self.assertEqual(expected_location, location) - - def test_add_auth_url_variations(self): - """ - Test that we can add an image via the swift backend with - a variety of different auth_address values - """ - swift_store_utils.is_multiple_swift_store_accounts_enabled = \ - mock.Mock(return_value=True) - conf = copy.deepcopy(SWIFT_CONF) - self.config(**conf) - - variations = { - 'store_4': 'swift+config://store_4/glance/%s', - 'store_5': 'swift+config://store_5/glance/%s', - 'store_6': 'swift+config://store_6/glance/%s' - } - - for variation, expected_location in variations.items(): - image_id = str(uuid.uuid4()) - expected_location = expected_location % image_id - expected_swift_size = FIVE_KB - expected_swift_contents = "*" * expected_swift_size - expected_checksum = \ - hashlib.md5(expected_swift_contents).hexdigest() - - image_swift = six.StringIO(expected_swift_contents) - - global SWIFT_PUT_OBJECT_CALLS - SWIFT_PUT_OBJECT_CALLS = 0 - conf['default_swift_reference'] = variation - self.config(**conf) - reload(swift) - self.store = Store() - location, size, checksum, _ = self.store.add(image_id, image_swift, - expected_swift_size) - - self.assertEqual(expected_location, location) - self.assertEqual(expected_swift_size, size) - self.assertEqual(expected_checksum, checksum) - self.assertEqual(SWIFT_PUT_OBJECT_CALLS, 1) - - loc = get_location_from_uri(expected_location) - (new_image_swift, new_image_size) = self.store.get(loc) - new_image_contents = ''.join([chunk for chunk in new_image_swift]) - new_image_swift_size = len(new_image_swift) - - self.assertEqual(expected_swift_contents, new_image_contents) - self.assertEqual(expected_swift_size, new_image_swift_size) - - def test_add_no_container_no_create(self): - """ - Tests that adding an image with a non-existing container - raises an appropriate exception - """ - conf = copy.deepcopy(SWIFT_CONF) - conf['swift_store_user'] = 'tenant:user' - conf['swift_store_create_container_on_put'] = False - conf['swift_store_container'] = 'noexist' - self.config(**conf) - reload(swift) - - self.store = Store() - - image_swift = six.StringIO("nevergonnamakeit") - - global SWIFT_PUT_OBJECT_CALLS - SWIFT_PUT_OBJECT_CALLS = 0 - - # We check the exception text to ensure the container - # missing text is found in it, otherwise, we would have - # simply used self.assertRaises here - exception_caught = False - try: - self.store.add(str(uuid.uuid4()), image_swift, 0) - except BackendException as e: - exception_caught = True - self.assertIn("container noexist does not exist " - "in Swift", utils.exception_to_str(e)) - self.assertTrue(exception_caught) - self.assertEqual(SWIFT_PUT_OBJECT_CALLS, 0) - - def test_add_no_container_and_create(self): - """ - Tests that adding an image with a non-existing container - creates the container automatically if flag is set - """ - swift_store_utils.is_multiple_swift_store_accounts_enabled = \ - mock.Mock(return_value=True) - expected_swift_size = FIVE_KB - expected_swift_contents = "*" * expected_swift_size - expected_checksum = hashlib.md5(expected_swift_contents).hexdigest() - expected_image_id = str(uuid.uuid4()) - loc = 'swift+config://ref1/noexist/%s' - expected_location = loc % (expected_image_id) - image_swift = six.StringIO(expected_swift_contents) - - global SWIFT_PUT_OBJECT_CALLS - SWIFT_PUT_OBJECT_CALLS = 0 - conf = copy.deepcopy(SWIFT_CONF) - conf['swift_store_user'] = 'tenant:user' - conf['swift_store_create_container_on_put'] = True - conf['swift_store_container'] = 'noexist' - self.config(**conf) - reload(swift) - self.store = Store() - location, size, checksum, _ = self.store.add(expected_image_id, - image_swift, - expected_swift_size) - - self.assertEqual(expected_location, location) - self.assertEqual(expected_swift_size, size) - self.assertEqual(expected_checksum, checksum) - self.assertEqual(SWIFT_PUT_OBJECT_CALLS, 1) - - loc = get_location_from_uri(expected_location) - (new_image_swift, new_image_size) = self.store.get(loc) - new_image_contents = ''.join([chunk for chunk in new_image_swift]) - new_image_swift_size = len(new_image_swift) - - self.assertEqual(expected_swift_contents, new_image_contents) - self.assertEqual(expected_swift_size, new_image_swift_size) - - def test_add_large_object(self): - """ - Tests that adding a very large image. We simulate the large - object by setting store.large_object_size to a small number - and then verify that there have been a number of calls to - put_object()... - """ - swift_store_utils.is_multiple_swift_store_accounts_enabled = \ - mock.Mock(return_value=True) - expected_swift_size = FIVE_KB - expected_swift_contents = "*" * expected_swift_size - expected_checksum = hashlib.md5(expected_swift_contents).hexdigest() - expected_image_id = str(uuid.uuid4()) - loc = 'swift+config://ref1/glance/%s' - expected_location = loc % (expected_image_id) - image_swift = six.StringIO(expected_swift_contents) - - global SWIFT_PUT_OBJECT_CALLS - SWIFT_PUT_OBJECT_CALLS = 0 - - self.store = Store() - orig_max_size = self.store.large_object_size - orig_temp_size = self.store.large_object_chunk_size - try: - self.store.large_object_size = 1024 - self.store.large_object_chunk_size = 1024 - location, size, checksum, _ = self.store.add(expected_image_id, - image_swift, - expected_swift_size) - finally: - self.store.large_object_chunk_size = orig_temp_size - self.store.large_object_size = orig_max_size - - self.assertEqual(expected_location, location) - self.assertEqual(expected_swift_size, size) - self.assertEqual(expected_checksum, checksum) - # Expecting 6 objects to be created on Swift -- 5 chunks and 1 - # manifest. - self.assertEqual(SWIFT_PUT_OBJECT_CALLS, 6) - - loc = get_location_from_uri(expected_location) - (new_image_swift, new_image_size) = self.store.get(loc) - new_image_contents = ''.join([chunk for chunk in new_image_swift]) - new_image_swift_size = len(new_image_contents) - - self.assertEqual(expected_swift_contents, new_image_contents) - self.assertEqual(expected_swift_size, new_image_swift_size) - - def test_add_large_object_zero_size(self): - """ - Tests that adding an image to Swift which has both an unknown size and - exceeds Swift's maximum limit of 5GB is correctly uploaded. - - We avoid the overhead of creating a 5GB object for this test by - temporarily setting MAX_SWIFT_OBJECT_SIZE to 1KB, and then adding - an object of 5KB. - - Bug lp:891738 - """ - # Set up a 'large' image of 5KB - expected_swift_size = FIVE_KB - expected_swift_contents = "*" * expected_swift_size - expected_checksum = hashlib.md5(expected_swift_contents).hexdigest() - expected_image_id = str(uuid.uuid4()) - loc = 'swift+config://ref1/glance/%s' - expected_location = loc % (expected_image_id) - image_swift = six.StringIO(expected_swift_contents) - - global SWIFT_PUT_OBJECT_CALLS - SWIFT_PUT_OBJECT_CALLS = 0 - - # Temporarily set Swift MAX_SWIFT_OBJECT_SIZE to 1KB and add our image, - # explicitly setting the image_length to 0 - - self.store = Store() - orig_max_size = self.store.large_object_size - orig_temp_size = self.store.large_object_chunk_size - global MAX_SWIFT_OBJECT_SIZE - orig_max_swift_object_size = MAX_SWIFT_OBJECT_SIZE - try: - MAX_SWIFT_OBJECT_SIZE = 1024 - self.store.large_object_size = 1024 - self.store.large_object_chunk_size = 1024 - location, size, checksum, _ = self.store.add(expected_image_id, - image_swift, 0) - finally: - self.store.large_object_chunk_size = orig_temp_size - self.store.large_object_size = orig_max_size - MAX_SWIFT_OBJECT_SIZE = orig_max_swift_object_size - - self.assertEqual(expected_location, location) - self.assertEqual(expected_swift_size, size) - self.assertEqual(expected_checksum, checksum) - # Expecting 7 calls to put_object -- 5 chunks, a zero chunk which is - # then deleted, and the manifest. Note the difference with above - # where the image_size is specified in advance (there's no zero chunk - # in that case). - self.assertEqual(SWIFT_PUT_OBJECT_CALLS, 7) - - loc = get_location_from_uri(expected_location) - (new_image_swift, new_image_size) = self.store.get(loc) - new_image_contents = ''.join([chunk for chunk in new_image_swift]) - new_image_swift_size = len(new_image_contents) - - self.assertEqual(expected_swift_contents, new_image_contents) - self.assertEqual(expected_swift_size, new_image_swift_size) - - def test_add_already_existing(self): - """ - Tests that adding an image with an existing identifier - raises an appropriate exception - """ - image_swift = six.StringIO("nevergonnamakeit") - self.assertRaises(exception.Duplicate, - self.store.add, - FAKE_UUID, image_swift, 0) - - def test_add_saves_and_reraises_and_not_uses_wildcard_raise(self): - image_id = str(uuid.uuid4()) - swift_size = self.store.large_object_size = 1024 - swift_contents = "*" * swift_size - connection = mock.Mock() - - def fake_delete_chunk(connection, - container, - chunks): - try: - raise Exception() - except Exception: - pass - - image_swift = six.StringIO(swift_contents) - connection.put_object.side_effect = exception.ClientConnectionError - self.store._delete_stale_chunks = fake_delete_chunk - - self.assertRaises(exception.ClientConnectionError, - self.store.add, - image_id, - image_swift, - swift_size, - connection) - - def _option_required(self, key): - conf = self.getConfig() - conf[key] = None - - try: - self.config(**conf) - self.store = Store() - return self.store.add == self.store.add_disabled - except Exception: - return False - return False - - def test_no_store_credentials(self): - """ - Tests that options without a valid credentials disables the add method - """ - swift.SWIFT_STORE_REF_PARAMS = {'ref1': {'auth_address': - 'authurl.com', 'user': '', - 'key': ''}} - self.store = Store() - self.assertEqual(self.store.add, self.store.add_disabled) - - def test_no_auth_address(self): - """ - Tests that options without auth address disables the add method - """ - swift.SWIFT_STORE_REF_PARAMS = {'ref1': {'auth_address': - '', 'user': 'user1', - 'key': 'key1'}} - self.store = Store() - self.assertEqual(self.store.add, self.store.add_disabled) - - def test_delete(self): - """ - Test we can delete an existing image in the swift store - """ - uri = "swift://%s:key@authurl/glance/%s" % ( - self.swift_store_user, FAKE_UUID) - loc = get_location_from_uri(uri) - self.store.delete(loc) - - self.assertRaises(exception.NotFound, self.store.get, loc) - - def test_delete_with_reference_params(self): - """ - Test we can delete an existing image in the swift store - """ - uri = "swift+config://ref1/glance/%s" % (FAKE_UUID) - loc = get_location_from_uri(uri) - self.store.delete(loc) - - self.assertRaises(exception.NotFound, self.store.get, loc) - - def test_delete_non_existing(self): - """ - Test that trying to delete a swift that doesn't exist - raises an error - """ - loc = get_location_from_uri("swift://%s:key@authurl/glance/noexist" % ( - self.swift_store_user)) - self.assertRaises(exception.NotFound, self.store.delete, loc) - - def test_read_acl_public(self): - """ - Test that we can set a public read acl. - """ - self.config(swift_store_multi_tenant=True) - context = glance.context.RequestContext() - store = Store(context) - uri = "swift+http://storeurl/glance/%s" % FAKE_UUID - loc = get_location_from_uri(uri) - store.set_acls(loc, public=True) - container_headers = swiftclient.client.head_container('x', 'y', - 'glance') - self.assertEqual(container_headers['X-Container-Read'], - ".r:*,.rlistings") - - def test_read_acl_tenants(self): - """ - Test that we can set read acl for tenants. - """ - self.config(swift_store_multi_tenant=True) - context = glance.context.RequestContext() - store = Store(context) - uri = "swift+http://storeurl/glance/%s" % FAKE_UUID - loc = get_location_from_uri(uri) - read_tenants = ['matt', 'mark'] - store.set_acls(loc, read_tenants=read_tenants) - container_headers = swiftclient.client.head_container('x', 'y', - 'glance') - self.assertEqual(container_headers['X-Container-Read'], - 'matt:*,mark:*') - - def test_write_acls(self): - """ - Test that we can set write acl for tenants. - """ - self.config(swift_store_multi_tenant=True) - context = glance.context.RequestContext() - store = Store(context) - uri = "swift+http://storeurl/glance/%s" % FAKE_UUID - loc = get_location_from_uri(uri) - read_tenants = ['frank', 'jim'] - store.set_acls(loc, write_tenants=read_tenants) - container_headers = swiftclient.client.head_container('x', 'y', - 'glance') - self.assertEqual(container_headers['X-Container-Write'], - 'frank:*,jim:*') - - -class TestStoreAuthV1(base.StoreClearingUnitTest, SwiftTests): - - def getConfig(self): - conf = SWIFT_CONF.copy() - conf['swift_store_auth_version'] = '1' - conf['swift_store_user'] = 'tenant:user1' - return conf - - def setUp(self): - """Establish a clean test environment""" - conf = self.getConfig() - self.config(**conf) - conf_file = 'glance-swift.conf' - self.test_dir = self.useFixture(fixtures.TempDir()).path - self.swift_config_file = self._copy_data_file(conf_file, self.test_dir) - conf.update({'swift_store_config_file': conf_file}) - self.config(swift_store_config_file=self.swift_config_file) - super(TestStoreAuthV1, self).setUp() - swift.SWIFT_STORE_REF_PARAMS = swift_store_utils.SwiftParams().params - self.stubs = stubout.StubOutForTesting() - stub_out_swiftclient(self.stubs, conf['swift_store_auth_version']) - self.store = Store() - self.addCleanup(self.stubs.UnsetAll) - - -class TestStoreAuthV2(TestStoreAuthV1): - - def getConfig(self): - conf = super(TestStoreAuthV2, self).getConfig() - conf['swift_store_auth_version'] = '2' - conf['swift_store_user'] = 'tenant:user1' - return conf - - def test_v2_with_no_tenant(self): - uri = "swift://failme:key@auth_address/glance/%s" % (FAKE_UUID) - loc = get_location_from_uri(uri) - self.assertRaises(exception.BadStoreUri, - self.store.get, - loc) - - def test_v2_multi_tenant_location(self): - conf = self.getConfig() - conf['swift_store_multi_tenant'] = True - uri = "swift://auth_address/glance/%s" % (FAKE_UUID) - loc = get_location_from_uri(uri) - self.assertEqual('swift', loc.store_name) - - -class FakeConnection(object): - def __init__(self, authurl, user, key, retries=5, preauthurl=None, - preauthtoken=None, snet=False, starting_backoff=1, - tenant_name=None, os_options=None, auth_version="1", - insecure=False, ssl_compression=True): - if os_options is None: - os_options = {} - - self.authurl = authurl - self.user = user - self.key = key - self.preauthurl = preauthurl - self.preauthtoken = preauthtoken - self.snet = snet - self.tenant_name = tenant_name - self.os_options = os_options - self.auth_version = auth_version - self.insecure = insecure - - -class TestSingleTenantStoreConnections(base.IsolatedUnitTest): - def setUp(self): - super(TestSingleTenantStoreConnections, self).setUp() - self.stubs.Set(swiftclient, 'Connection', FakeConnection) - self.store = glance.store.swift.SingleTenantStore() - specs = {'scheme': 'swift', - 'auth_or_store_url': 'example.com/v2/', - 'user': 'tenant:user1', - 'key': 'key1', - 'container': 'cont', - 'obj': 'object'} - self.location = glance.store.swift.StoreLocation(specs) - - def test_basic_connection(self): - connection = self.store.get_connection(self.location) - self.assertEqual(connection.authurl, 'https://example.com/v2/') - self.assertEqual(connection.auth_version, '2') - self.assertEqual(connection.user, 'user1') - self.assertEqual(connection.tenant_name, 'tenant') - self.assertFalse(connection.snet) - self.assertEqual(connection.key, 'key1') - self.assertIsNone(connection.preauthurl) - self.assertIsNone(connection.preauthtoken) - self.assertFalse(connection.insecure) - self.assertEqual(connection.os_options, - {'service_type': 'object-store', - 'endpoint_type': 'publicURL'}) - - def test_connection_with_no_trailing_slash(self): - self.location.auth_or_store_url = 'example.com/v2' - connection = self.store.get_connection(self.location) - self.assertEqual(connection.authurl, 'https://example.com/v2/') - - def test_connection_insecure(self): - self.config(swift_store_auth_insecure=True) - self.store.configure() - connection = self.store.get_connection(self.location) - self.assertTrue(connection.insecure) - - def test_connection_with_auth_v1(self): - self.config(swift_store_auth_version='1') - self.store.configure() - self.location.user = 'auth_v1_user' - connection = self.store.get_connection(self.location) - self.assertEqual(connection.auth_version, '1') - self.assertEqual(connection.user, 'auth_v1_user') - self.assertIsNone(connection.tenant_name) - - def test_connection_invalid_user(self): - self.store.configure() - self.location.user = 'invalid:format:user' - self.assertRaises(exception.BadStoreUri, - self.store.get_connection, self.location) - - def test_connection_missing_user(self): - self.store.configure() - self.location.user = None - self.assertRaises(exception.BadStoreUri, - self.store.get_connection, self.location) - - def test_connection_with_region(self): - self.config(swift_store_region='Sahara') - self.store.configure() - connection = self.store.get_connection(self.location) - self.assertEqual(connection.os_options, - {'region_name': 'Sahara', - 'service_type': 'object-store', - 'endpoint_type': 'publicURL'}) - - def test_connection_with_service_type(self): - self.config(swift_store_service_type='shoe-store') - self.store.configure() - connection = self.store.get_connection(self.location) - self.assertEqual(connection.os_options, - {'service_type': 'shoe-store', - 'endpoint_type': 'publicURL'}) - - def test_connection_with_endpoint_type(self): - self.config(swift_store_endpoint_type='internalURL') - self.store.configure() - connection = self.store.get_connection(self.location) - self.assertEqual(connection.os_options, - {'service_type': 'object-store', - 'endpoint_type': 'internalURL'}) - - def test_connection_with_snet(self): - self.config(swift_enable_snet=True) - self.store.configure() - connection = self.store.get_connection(self.location) - self.assertTrue(connection.snet) - - def test_bad_location_uri(self): - self.store.configure() - self.location.uri = 'http://bad_uri://' - self.assertRaises(exception.BadStoreUri, - self.location.parse_uri, - self.location.uri) - - def test_bad_location_uri_invalid_credentials(self): - self.store.configure() - self.location.uri = 'swift://bad_creds@uri/cont/obj' - self.assertRaises(exception.BadStoreUri, - self.location.parse_uri, - self.location.uri) - - def test_bad_location_uri_invalid_object_path(self): - self.store.configure() - self.location.uri = 'swift://user:key@uri/cont' - self.assertRaises(exception.BadStoreUri, - self.location.parse_uri, - self.location.uri) - - -class TestMultiTenantStoreConnections(base.IsolatedUnitTest): - def setUp(self): - super(TestMultiTenantStoreConnections, self).setUp() - self.stubs.Set(swiftclient, 'Connection', FakeConnection) - self.context = glance.context.RequestContext( - user='tenant:user1', tenant='tenant', auth_tok='0123') - self.store = glance.store.swift.MultiTenantStore(self.context) - specs = {'scheme': 'swift', - 'auth_or_store_url': 'example.com', - 'container': 'cont', - 'obj': 'object'} - self.location = glance.store.swift.StoreLocation(specs) - - def test_basic_connection(self): - self.store.configure() - connection = self.store.get_connection(self.location) - self.assertIsNone(connection.authurl) - self.assertEqual(connection.auth_version, '2') - self.assertEqual(connection.user, 'tenant:user1') - self.assertEqual(connection.tenant_name, 'tenant') - self.assertIsNone(connection.key) - self.assertFalse(connection.snet) - self.assertEqual(connection.preauthurl, 'https://example.com') - self.assertEqual(connection.preauthtoken, '0123') - self.assertEqual(connection.os_options, {}) - - def test_connection_with_snet(self): - self.config(swift_enable_snet=True) - self.store.configure() - connection = self.store.get_connection(self.location) - self.assertTrue(connection.snet) - - -class FakeGetEndpoint(object): - def __init__(self, response): - self.response = response - - def __call__(self, service_catalog, service_type=None, - endpoint_region=None, endpoint_type=None): - self.service_type = service_type - self.endpoint_region = endpoint_region - self.endpoint_type = endpoint_type - return self.response - - -class TestCreatingLocations(base.IsolatedUnitTest): - def setUp(self): - conf = copy.deepcopy(SWIFT_CONF) - self.config(**conf) - reload(swift) - super(TestCreatingLocations, self).setUp() - - def test_single_tenant_location(self): - conf = copy.deepcopy(SWIFT_CONF) - conf['swift_store_container'] = 'container' - conf_file = "glance-swift.conf" - test_dir = self.useFixture(fixtures.TempDir()).path - self.swift_config_file = self._copy_data_file(conf_file, test_dir) - conf.update({'swift_store_config_file': self.swift_config_file}) - conf['default_swift_reference'] = 'ref1' - self.config(**conf) - reload(swift) - - store = swift.SingleTenantStore() - location = store.create_location('image-id') - self.assertEqual(location.scheme, 'swift+https') - self.assertEqual(location.swift_url, 'https://example.com') - self.assertEqual(location.container, 'container') - self.assertEqual(location.obj, 'image-id') - self.assertEqual(location.user, 'tenant:user1') - self.assertEqual(location.key, 'key1') - - def test_single_tenant_location_http(self): - conf_file = "glance-swift.conf" - test_dir = self.useFixture(fixtures.TempDir()).path - self.swift_config_file = self._copy_data_file(conf_file, test_dir) - self.config(swift_store_container='container', - default_swift_reference='ref2', - swift_store_config_file=self.swift_config_file) - swift.SWIFT_STORE_REF_PARAMS = swift_store_utils.SwiftParams().params - store = glance.store.swift.SingleTenantStore() - location = store.create_location('image-id') - self.assertEqual(location.scheme, 'swift+http') - self.assertEqual(location.swift_url, 'http://example.com') - - def test_multi_tenant_location(self): - self.config(swift_store_container='container') - fake_get_endpoint = FakeGetEndpoint('https://some_endpoint') - self.stubs.Set(glance.common.auth, 'get_endpoint', fake_get_endpoint) - context = glance.context.RequestContext( - user='user', tenant='tenant', auth_tok='123', - service_catalog={}) - store = glance.store.swift.MultiTenantStore(context) - location = store.create_location('image-id') - self.assertEqual(location.scheme, 'swift+https') - self.assertEqual(location.swift_url, 'https://some_endpoint') - self.assertEqual(location.container, 'container_image-id') - self.assertEqual(location.obj, 'image-id') - self.assertIsNone(location.user) - self.assertIsNone(location.key) - self.assertEqual(fake_get_endpoint.service_type, 'object-store') - - def test_multi_tenant_location_http(self): - fake_get_endpoint = FakeGetEndpoint('http://some_endpoint') - self.stubs.Set(glance.common.auth, 'get_endpoint', fake_get_endpoint) - context = glance.context.RequestContext( - user='user', tenant='tenant', auth_tok='123', - service_catalog={}) - store = glance.store.swift.MultiTenantStore(context) - location = store.create_location('image-id') - self.assertEqual(location.scheme, 'swift+http') - self.assertEqual(location.swift_url, 'http://some_endpoint') - - def test_multi_tenant_location_with_region(self): - self.config(swift_store_region='WestCarolina') - fake_get_endpoint = FakeGetEndpoint('https://some_endpoint') - self.stubs.Set(glance.common.auth, 'get_endpoint', fake_get_endpoint) - context = glance.context.RequestContext( - user='user', tenant='tenant', auth_tok='123', - service_catalog={}) - glance.store.swift.MultiTenantStore(context) - self.assertEqual(fake_get_endpoint.endpoint_region, 'WestCarolina') - - def test_multi_tenant_location_custom_service_type(self): - self.config(swift_store_service_type='toy-store') - fake_get_endpoint = FakeGetEndpoint('https://some_endpoint') - self.stubs.Set(glance.common.auth, 'get_endpoint', fake_get_endpoint) - context = glance.context.RequestContext( - user='user', tenant='tenant', auth_tok='123', - service_catalog={}) - glance.store.swift.MultiTenantStore(context) - self.assertEqual(fake_get_endpoint.service_type, 'toy-store') - - def test_multi_tenant_location_custom_endpoint_type(self): - self.config(swift_store_endpoint_type='InternalURL') - fake_get_endpoint = FakeGetEndpoint('https://some_endpoint') - self.stubs.Set(glance.common.auth, 'get_endpoint', fake_get_endpoint) - context = glance.context.RequestContext( - user='user', tenant='tenant', auth_tok='123', - service_catalog={}) - glance.store.swift.MultiTenantStore(context) - self.assertEqual(fake_get_endpoint.endpoint_type, 'InternalURL') - - -class TestChunkReader(base.StoreClearingUnitTest): - def setUp(self): - conf = copy.deepcopy(SWIFT_CONF) - self.config(**conf) - super(TestChunkReader, self).setUp() - - def test_read_all_data(self): - """ - Replicate what goes on in the Swift driver with the - repeated creation of the ChunkReader object - """ - CHUNKSIZE = 100 - checksum = hashlib.md5() - data_file = tempfile.NamedTemporaryFile() - data_file.write('*' * units.Ki) - data_file.flush() - infile = open(data_file.name, 'rb') - bytes_read = 0 - while True: - cr = glance.store.swift.ChunkReader(infile, checksum, CHUNKSIZE) - chunk = cr.read(CHUNKSIZE) - bytes_read += len(chunk) - if not chunk: - break - self.assertEqual(1024, bytes_read) - data_file.close() diff --git a/glance/tests/unit/test_vmware_store.py b/glance/tests/unit/test_vmware_store.py deleted file mode 100644 index 7d12e0c01a..0000000000 --- a/glance/tests/unit/test_vmware_store.py +++ /dev/null @@ -1,394 +0,0 @@ -# Copyright 2014 OpenStack, LLC -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Tests the VMware Datastore backend store""" - -import hashlib -import uuid - -import mock -import six - -from glance.common import exception -from glance.openstack.common import units -from glance.store.location import get_location_from_uri -import glance.store.vmware_datastore as vm_store -from glance.store.vmware_datastore import Store -from glance.tests.unit import base -from glance.tests.unit import utils as unit_utils -from glance.tests import utils - - -FAKE_UUID = str(uuid.uuid4()) - -FIVE_KB = 5 * units.Ki - -VMWARE_DATASTORE_CONF = { - 'verbose': True, - 'debug': True, - 'known_stores': ['glance.store.vmware_datastore.Store'], - 'default_store': 'vsphere', - 'vmware_server_host': '127.0.0.1', - 'vmware_server_username': 'username', - 'vmware_server_password': 'password', - 'vmware_datacenter_path': 'dc1', - 'vmware_datastore_name': 'ds1', - 'vmware_store_image_dir': '/openstack_glance', - 'vmware_api_insecure': 'True', - 'vmware_api_retry_count': 10 -} - - -def format_location(host_ip, folder_name, - image_id, datacenter_path, datastore_name): - """ - Helper method that returns a VMware Datastore store URI given - the component pieces. - """ - scheme = 'vsphere' - return ("%s://%s/folder%s/%s?dsName=%s&dcPath=%s" - % (scheme, host_ip, folder_name, - image_id, datastore_name, datacenter_path)) - - -class FakeHTTPConnection(object): - - def __init__(self, status=200, *args, **kwargs): - self.status = status - pass - - def getresponse(self): - return utils.FakeHTTPResponse(status=self.status) - - def request(self, *_args, **_kwargs): - pass - - def close(self): - pass - - -class TestStore(base.StoreClearingUnitTest): - - @mock.patch('oslo.vmware.api.VMwareAPISession', autospec=True) - def setUp(self, mock_session): - """Establish a clean test environment""" - - self.config(default_store='file') - - # NOTE(flaper87): Each store should test - # this in their test suite. - self.config(known_stores=VMWARE_DATASTORE_CONF['known_stores']) - - super(TestStore, self).setUp() - - Store.READ_CHUNKSIZE = 2 - self.store = Store() - - class FakeSession: - def __init__(self): - self.vim = FakeVim() - - class FakeVim: - def __init__(self): - self.client = FakeClient() - - class FakeClient: - def __init__(self): - self.options = FakeOptions() - - class FakeOptions: - def __init__(self): - self.transport = FakeTransport() - - class FakeTransport: - def __init__(self): - self.cookiejar = FakeCookieJar() - - class FakeCookieJar: - pass - - self.store.scheme = VMWARE_DATASTORE_CONF['default_store'] - self.store.server_host = ( - VMWARE_DATASTORE_CONF['vmware_server_host']) - self.store.datacenter_path = ( - VMWARE_DATASTORE_CONF['vmware_datacenter_path']) - self.store.datastore_name = ( - VMWARE_DATASTORE_CONF['vmware_datastore_name']) - self.store.api_insecure = ( - VMWARE_DATASTORE_CONF['vmware_api_insecure']) - self.store.api_retry_count = ( - VMWARE_DATASTORE_CONF['vmware_api_retry_count']) - self.store._session = FakeSession() - self.store._session.invoke_api = mock.Mock() - self.store._session.wait_for_task = mock.Mock() - - self.store.store_image_dir = ( - VMWARE_DATASTORE_CONF['vmware_store_image_dir']) - Store._build_vim_cookie_header = mock.Mock() - self.addCleanup(self.stubs.UnsetAll) - - def test_get(self): - """Test a "normal" retrieval of an image in chunks""" - expected_image_size = 31 - expected_returns = ['I ', 'am', ' a', ' t', 'ea', 'po', 't,', ' s', - 'ho', 'rt', ' a', 'nd', ' s', 'to', 'ut', '\n'] - loc = get_location_from_uri( - "vsphere://127.0.0.1/folder/openstack_glance/%s" - "?dsName=ds1&dcPath=dc1" % FAKE_UUID) - with mock.patch('httplib.HTTPConnection') as HttpConn: - HttpConn.return_value = FakeHTTPConnection() - (image_file, image_size) = self.store.get(loc) - self.assertEqual(image_size, expected_image_size) - chunks = [c for c in image_file] - self.assertEqual(chunks, expected_returns) - - def test_get_non_existing(self): - """ - Test that trying to retrieve an image that doesn't exist - raises an error - """ - loc = get_location_from_uri("vsphere://127.0.0.1/folder/openstack_glan" - "ce/%s?dsName=ds1&dcPath=dc1" % FAKE_UUID) - with mock.patch('httplib.HTTPConnection') as HttpConn: - HttpConn.return_value = FakeHTTPConnection(status=404) - self.assertRaises(exception.NotFound, self.store.get, loc) - - @mock.patch.object(vm_store._Reader, 'size') - def test_add(self, fake_size): - """Test that we can add an image via the VMware backend""" - expected_image_id = str(uuid.uuid4()) - expected_size = FIVE_KB - expected_contents = "*" * expected_size - hash_code = hashlib.md5(expected_contents) - expected_checksum = hash_code.hexdigest() - fake_size.__get__ = mock.Mock(return_value=expected_size) - with mock.patch('hashlib.md5') as md5: - md5.return_value = hash_code - expected_location = format_location( - VMWARE_DATASTORE_CONF['vmware_server_host'], - VMWARE_DATASTORE_CONF['vmware_store_image_dir'], - expected_image_id, - VMWARE_DATASTORE_CONF['vmware_datacenter_path'], - VMWARE_DATASTORE_CONF['vmware_datastore_name']) - image = six.StringIO(expected_contents) - with mock.patch('httplib.HTTPConnection') as HttpConn: - HttpConn.return_value = FakeHTTPConnection() - location, size, checksum, _ = self.store.add(expected_image_id, - image, - expected_size) - self.assertEqual(unit_utils.sort_url_by_qs_keys(expected_location), - unit_utils.sort_url_by_qs_keys(location)) - self.assertEqual(expected_size, size) - self.assertEqual(expected_checksum, checksum) - - @mock.patch.object(vm_store._Reader, 'size') - def test_add_size_zero(self, fake_size): - """ - Test that when specifying size zero for the image to add, - the actual size of the image is returned. - """ - expected_image_id = str(uuid.uuid4()) - expected_size = FIVE_KB - expected_contents = "*" * expected_size - hash_code = hashlib.md5(expected_contents) - expected_checksum = hash_code.hexdigest() - fake_size.__get__ = mock.Mock(return_value=expected_size) - with mock.patch('hashlib.md5') as md5: - md5.return_value = hash_code - expected_location = format_location( - VMWARE_DATASTORE_CONF['vmware_server_host'], - VMWARE_DATASTORE_CONF['vmware_store_image_dir'], - expected_image_id, - VMWARE_DATASTORE_CONF['vmware_datacenter_path'], - VMWARE_DATASTORE_CONF['vmware_datastore_name']) - image = six.StringIO(expected_contents) - with mock.patch('httplib.HTTPConnection') as HttpConn: - HttpConn.return_value = FakeHTTPConnection() - location, size, checksum, _ = self.store.add(expected_image_id, - image, 0) - self.assertEqual(unit_utils.sort_url_by_qs_keys(expected_location), - unit_utils.sort_url_by_qs_keys(location)) - self.assertEqual(expected_size, size) - self.assertEqual(expected_checksum, checksum) - - def test_delete(self): - """Test we can delete an existing image in the VMware store""" - loc = get_location_from_uri( - "vsphere://127.0.0.1/folder/openstack_glance/%s?" - "dsName=ds1&dcPath=dc1" % FAKE_UUID) - with mock.patch('httplib.HTTPConnection') as HttpConn: - HttpConn.return_value = FakeHTTPConnection() - Store._service_content = mock.Mock() - self.store.delete(loc) - with mock.patch('httplib.HTTPConnection') as HttpConn: - HttpConn.return_value = FakeHTTPConnection(status=404) - self.assertRaises(exception.NotFound, self.store.get, loc) - - def test_get_size(self): - """Test we can get the size of an existing image in the VMware store""" - loc = get_location_from_uri( - "vsphere://127.0.0.1/folder/openstack_glance/%s" - "?dsName=ds1&dcPath=dc1" % FAKE_UUID) - with mock.patch('httplib.HTTPConnection') as HttpConn: - HttpConn.return_value = FakeHTTPConnection() - image_size = self.store.get_size(loc) - self.assertEqual(image_size, 31) - - def test_get_size_non_existing(self): - """ - Test that trying to retrieve an image size that doesn't exist - raises an error - """ - loc = get_location_from_uri("vsphere://127.0.0.1/folder/openstack_glan" - "ce/%s?dsName=ds1&dcPath=dc1" % FAKE_UUID) - with mock.patch('httplib.HTTPConnection') as HttpConn: - HttpConn.return_value = FakeHTTPConnection(status=404) - self.assertRaises(exception.NotFound, self.store.get_size, loc) - - def test_reader_full(self): - content = 'XXX' - image = six.StringIO(content) - expected_checksum = hashlib.md5(content).hexdigest() - reader = vm_store._Reader(image) - ret = reader.read() - self.assertEqual(content, ret) - self.assertEqual(expected_checksum, reader.checksum.hexdigest()) - self.assertEqual(len(content), reader.size) - - def test_reader_partial(self): - content = 'XXX' - image = six.StringIO(content) - expected_checksum = hashlib.md5('X').hexdigest() - reader = vm_store._Reader(image) - ret = reader.read(1) - self.assertEqual('X', ret) - self.assertEqual(expected_checksum, reader.checksum.hexdigest()) - self.assertEqual(1, reader.size) - - def test_rewind(self): - content = 'XXX' - image = six.StringIO(content) - expected_checksum = hashlib.md5(content).hexdigest() - reader = vm_store._Reader(image) - reader.read(1) - ret = reader.read() - self.assertEqual('XX', ret) - self.assertEqual(expected_checksum, reader.checksum.hexdigest()) - self.assertEqual(len(content), reader.size) - reader.rewind() - ret = reader.read() - self.assertEqual(content, ret) - self.assertEqual(expected_checksum, reader.checksum.hexdigest()) - self.assertEqual(len(content), reader.size) - - def test_chunkreader_image_fits_in_blocksize(self): - """ - Test that the image file reader returns the expected chunk of data - when the block size is larger than the image. - """ - content = 'XXX' - image = six.StringIO(content) - expected_checksum = hashlib.md5(content).hexdigest() - reader = vm_store._ChunkReader(image) - ret = reader.read() - expected_chunk = '%x\r\n%s\r\n' % (len(content), content) - last_chunk = '0\r\n\r\n' - self.assertEqual('%s%s' % (expected_chunk, last_chunk), ret) - self.assertEqual(image.len, reader.size) - self.assertEqual(expected_checksum, reader.checksum.hexdigest()) - self.assertTrue(reader.closed) - ret = reader.read() - self.assertEqual(image.len, reader.size) - self.assertEqual(expected_checksum, reader.checksum.hexdigest()) - self.assertTrue(reader.closed) - self.assertEqual('', ret) - - def test_chunkreader_image_larger_blocksize(self): - """ - Test that the image file reader returns the expected chunks when - the block size specified is smaller than the image. - """ - content = 'XXX' - image = six.StringIO(content) - expected_checksum = hashlib.md5(content).hexdigest() - last_chunk = '0\r\n\r\n' - reader = vm_store._ChunkReader(image, blocksize=1) - ret = reader.read() - expected_chunk = '1\r\nX\r\n' - self.assertEqual('%s%s%s%s' % (expected_chunk, expected_chunk, - expected_chunk, last_chunk), ret) - self.assertEqual(expected_checksum, reader.checksum.hexdigest()) - self.assertEqual(image.len, reader.size) - self.assertTrue(reader.closed) - - def test_chunkreader_size(self): - """Test that the image reader takes into account the specified size.""" - content = 'XXX' - image = six.StringIO(content) - expected_checksum = hashlib.md5(content).hexdigest() - reader = vm_store._ChunkReader(image, blocksize=1) - ret = reader.read(size=3) - self.assertEqual('1\r\n', ret) - ret = reader.read(size=1) - self.assertEqual('X', ret) - ret = reader.read() - self.assertEqual(expected_checksum, reader.checksum.hexdigest()) - self.assertEqual(image.len, reader.size) - self.assertTrue(reader.closed) - - def test_sanity_check_api_retry_count(self): - """Test that sanity check raises if api_retry_count is <= 0.""" - vm_store.CONF.vmware_api_retry_count = -1 - self.assertRaises(exception.BadStoreConfiguration, - self.store._sanity_check) - vm_store.CONF.vmware_api_retry_count = 0 - self.assertRaises(exception.BadStoreConfiguration, - self.store._sanity_check) - vm_store.CONF.vmware_api_retry_count = 1 - try: - self.store._sanity_check() - except exception.BadStoreConfiguration: - self.fail() - - def test_sanity_check_task_poll_interval(self): - """Test that sanity check raises if task_poll_interval is <= 0.""" - vm_store.CONF.vmware_task_poll_interval = -1 - self.assertRaises(exception.BadStoreConfiguration, - self.store._sanity_check) - vm_store.CONF.vmware_task_poll_interval = 0 - self.assertRaises(exception.BadStoreConfiguration, - self.store._sanity_check) - vm_store.CONF.vmware_task_poll_interval = 1 - try: - self.store._sanity_check() - except exception.BadStoreConfiguration: - self.fail() - - def test_retry_count(self): - expected_image_id = str(uuid.uuid4()) - expected_size = FIVE_KB - expected_contents = "*" * expected_size - image = six.StringIO(expected_contents) - self.store._create_session = mock.Mock() - with mock.patch('httplib.HTTPConnection') as HttpConn: - HttpConn.return_value = FakeHTTPConnection(status=401) - try: - location, size, checksum, _ = self.store.add(expected_image_id, - image, - expected_size) - except exception.NotAuthenticated: - pass - self.assertEqual(VMWARE_DATASTORE_CONF['vmware_api_retry_count'] + 1, - self.store._create_session.call_count) diff --git a/glance/tests/unit/utils.py b/glance/tests/unit/utils.py index 2c1bc5291e..ac27a29285 100644 --- a/glance/tests/unit/utils.py +++ b/glance/tests/unit/utils.py @@ -16,6 +16,7 @@ import urllib import urlparse +import glance_store as store from oslo.config import cfg from glance.common import exception @@ -23,7 +24,6 @@ from glance.common import wsgi import glance.context import glance.db.simple.api as simple_db import glance.openstack.common.log as logging -import glance.store CONF = cfg.CONF @@ -78,7 +78,7 @@ def get_fake_request(path='', method='POST', is_admin=False, user=USER1, return req -def fake_get_size_from_backend(context, uri): +def fake_get_size_from_backend(uri, context=None): return 1 @@ -151,8 +151,8 @@ class FakeStoreAPI(object): def create_stores(self): pass - def set_acls(self, context, uri, public=False, - read_tenants=None, write_tenants=None): + def set_acls(self, uri, public=False, read_tenants=None, + write_tenants=None, context=None): if read_tenants is None: read_tenants = [] if write_tenants is None: @@ -164,19 +164,20 @@ class FakeStoreAPI(object): 'write': write_tenants, } - def get_from_backend(self, context, location): + def get_from_backend(self, location, context=None): try: scheme = location[:location.find('/') - 1] if scheme == 'unknown': - raise exception.UnknownScheme(scheme=scheme) + raise store.UnknownScheme(scheme=scheme) return self.data[location] except KeyError: - raise exception.NotFound() + raise store.NotFound() - def get_size_from_backend(self, context, location): - return self.get_from_backend(context, location)[1] + def get_size_from_backend(self, location, context=None): + return self.get_from_backend(location, context=context)[1] - def add_to_backend(self, context, scheme, image_id, data, size): + def add_to_backend(self, conf, image_id, data, size, + scheme=None, context=None): store_max_size = 7 current_store_size = 2 for location in self.data.keys(): @@ -198,7 +199,7 @@ class FakeStoreAPI(object): return (image_id, size, checksum, self.store_metadata) def check_location_metadata(self, val, key=''): - glance.store.check_location_metadata(val) + store.check_location_metadata(val) class FakePolicyEnforcer(object): diff --git a/glance/tests/unit/v1/test_api.py b/glance/tests/unit/v1/test_api.py index 7192efe869..f5c59a1fc9 100644 --- a/glance/tests/unit/v1/test_api.py +++ b/glance/tests/unit/v1/test_api.py @@ -20,6 +20,7 @@ import datetime import hashlib import uuid +import glance_store as store import mock from oslo.config import cfg import routes @@ -39,8 +40,6 @@ from glance.openstack.common import jsonutils from glance.openstack.common import timeutils import glance.registry.client.v1.api as registry -import glance.store.filesystem -from glance.store import http from glance.tests.unit import base import glance.tests.unit.utils as unit_test_utils from glance.tests import utils as test_utils @@ -91,7 +90,7 @@ class TestGlanceAPI(base.IsolatedUnitTest): 'metadata': {}, 'status': 'active'}], 'properties': {}}] self.context = glance.context.RequestContext(is_admin=True) - glance.api.v1.images.validate_location = mock.Mock() + store.validate_location = mock.Mock() db_api.get_engine() self.destroy_fixtures() self.create_fixtures() @@ -126,7 +125,9 @@ class TestGlanceAPI(base.IsolatedUnitTest): for k, v in six.iteritems(fixture_headers): req.headers[k] = v - with mock.patch.object(http.Store, 'get_size') as mocked_size: + http = store.get_store_from_scheme('http') + + with mock.patch.object(http, 'get_size') as mocked_size: mocked_size.return_value = 0 res = req.get_response(self.api) self.assertEqual(res.status_int, 201) @@ -246,7 +247,8 @@ class TestGlanceAPI(base.IsolatedUnitTest): for k, v in six.iteritems(fixture_headers): req.headers[k] = v - with mock.patch.object(http.Store, 'get_size') as mocked_size: + http = store.get_store_from_scheme('http') + with mock.patch.object(http, 'get_size') as mocked_size: mocked_size.return_value = 0 res = req.get_response(self.api) self.assertEqual(res.status_int, 201) @@ -284,7 +286,9 @@ class TestGlanceAPI(base.IsolatedUnitTest): for k, v in six.iteritems(fixture_headers): req.headers[k] = v - with mock.patch.object(http.Store, 'get_size') as mocked_size: + http = store.get_store_from_scheme('http') + + with mock.patch.object(http, 'get_size') as mocked_size: mocked_size.return_value = 0 res = req.get_response(self.api) self.assertEqual(res.status_int, 201) @@ -341,7 +345,9 @@ class TestGlanceAPI(base.IsolatedUnitTest): for k, v in six.iteritems(fixture_headers): req.headers[k] = v - with mock.patch.object(http.Store, 'get_size') as mocked_size: + http = store.get_store_from_scheme('http') + + with mock.patch.object(http, 'get_size') as mocked_size: mocked_size.return_value = 0 res = req.get_response(self.api) self.assertEqual(res.status_int, 400) @@ -384,7 +390,7 @@ class TestGlanceAPI(base.IsolatedUnitTest): def test_create_with_location_bad_store_uri(self): fixture_headers = { - 'x-image-meta-store': 'swift', + 'x-image-meta-store': 'file', 'x-image-meta-name': 'bogus', 'x-image-meta-location': 'http://', 'x-image-meta-disk-format': 'qcow2', @@ -495,7 +501,9 @@ class TestGlanceAPI(base.IsolatedUnitTest): req.method = 'PUT' req.headers['x-image-meta-location'] = 'http://localhost:0/images/123' - with mock.patch.object(http.Store, 'get_size') as mocked_size: + http = store.get_store_from_scheme('http') + + with mock.patch.object(http, 'get_size') as mocked_size: mocked_size.return_value = 0 res = req.get_response(self.api) self.assertEqual(res.status_int, 200) @@ -953,7 +961,10 @@ class TestGlanceAPI(base.IsolatedUnitTest): req = webob.Request.blank("/images") req.headers['Content-Type'] = 'application/octet-stream' req.method = 'POST' - with mock.patch.object(http.Store, 'get_size') as size: + + http = store.get_store_from_scheme('http') + + with mock.patch.object(http, 'get_size') as size: size.return_value = 2 for k, v in six.iteritems(fixture_headers): @@ -966,8 +977,8 @@ class TestGlanceAPI(base.IsolatedUnitTest): """Tests creates an image from location and conflict image size""" mock_validate_location = mock.Mock() - glance.api.v1.images.validate_location = mock_validate_location - mock_validate_location.side_effect = exception.BadStoreUri() + store.validate_location = mock_validate_location + mock_validate_location.side_effect = store.BadStoreUri() fixture_headers = {'x-image-meta-store': 'file', 'x-image-meta-disk-format': 'vhd', @@ -1240,26 +1251,29 @@ class TestGlanceAPI(base.IsolatedUnitTest): self.assertEqual(res.status_int, 200) self.assertEqual("deleted", res.headers['x-image-meta-status']) - @mock.patch.object(glance.store.filesystem.Store, 'delete') - def test_image_status_when_delete_fails(self, mock_fsstore_delete): + def test_image_status_when_delete_fails(self): """ Tests that the image status set to active if deletion of image fails. """ - mock_fsstore_delete.side_effect = exception.Forbidden() - # trigger the v1 delete api - req = webob.Request.blank("/images/%s" % UUID2) - req.method = 'DELETE' - res = req.get_response(self.api) - self.assertEqual(res.status_int, 403) - self.assertTrue('Forbidden to delete image' in res.body) + fs = store.get_store_from_scheme('file') - # check image metadata is still there with active state - req = webob.Request.blank("/images/%s" % UUID2) - req.method = 'HEAD' - res = req.get_response(self.api) - self.assertEqual(res.status_int, 200) - self.assertEqual("active", res.headers['x-image-meta-status']) + with mock.patch.object(fs, 'delete') as mock_fsstore_delete: + mock_fsstore_delete.side_effect = exception.Forbidden() + + # trigger the v1 delete api + req = webob.Request.blank("/images/%s" % UUID2) + req.method = 'DELETE' + res = req.get_response(self.api) + self.assertEqual(res.status_int, 403) + self.assertTrue('Forbidden to delete image' in res.body) + + # check image metadata is still there with active state + req = webob.Request.blank("/images/%s" % UUID2) + req.method = 'HEAD' + res = req.get_response(self.api) + self.assertEqual(res.status_int, 200) + self.assertEqual("active", res.headers['x-image-meta-status']) def test_delete_pending_delete_image(self): """ @@ -1542,7 +1556,7 @@ class TestGlanceAPI(base.IsolatedUnitTest): # We expect 500 since an exception occured during upload. self.assertEqual(500, res.status_int) - @mock.patch('glance.store.store_add_to_backend') + @mock.patch('glance_store.store_add_to_backend') def test_upload_safe_kill(self, mock_store_add_to_backend): def mock_store_add_to_backend_w_exception(*args, **kwargs): @@ -1562,7 +1576,7 @@ class TestGlanceAPI(base.IsolatedUnitTest): self.assertEqual(1, mock_store_add_to_backend.call_count) - @mock.patch('glance.store.store_add_to_backend') + @mock.patch('glance_store.store_add_to_backend') def test_upload_safe_kill_deleted(self, mock_store_add_to_backend): test_router_api = router.API(self.mapper) self.api = test_utils.FakeAuthMiddleware(test_router_api, @@ -2296,7 +2310,9 @@ class TestGlanceAPI(base.IsolatedUnitTest): req.headers[k] = v req.method = 'POST' - with mock.patch.object(http.Store, 'get_size') as size: + http = store.get_store_from_scheme('http') + + with mock.patch.object(http, 'get_size') as size: size.return_value = 0 res = req.get_response(self.api) self.assertEqual(res.status_int, 201) @@ -2616,15 +2632,6 @@ class TestGlanceAPI(base.IsolatedUnitTest): res = req.get_response(self.api) self.assertEqual(res.status_int, 403) - @mock.patch.object(glance.store.filesystem.Store, 'delete') - def test_delete_image_in_use(self, mock_filesystem_delete): - mock_filesystem_delete.side_effect = exception.InUseByStore() - - req = webob.Request.blank("/images/%s" % UUID2) - req.method = 'DELETE' - res = req.get_response(self.api) - self.assertEqual(res.status_int, 409) - def test_head_details(self): req = webob.Request.blank('/images/detail') req.method = 'HEAD' diff --git a/glance/tests/unit/v1/test_registry_api.py b/glance/tests/unit/v1/test_registry_api.py index 1132555869..71c6d4b4e1 100644 --- a/glance/tests/unit/v1/test_registry_api.py +++ b/glance/tests/unit/v1/test_registry_api.py @@ -34,7 +34,6 @@ from glance.openstack.common import jsonutils from glance.openstack.common import timeutils from glance.registry.api import v1 as rserver -import glance.store.filesystem from glance.tests.unit import base from glance.tests import utils as test_utils diff --git a/glance/tests/unit/v1/test_upload_utils.py b/glance/tests/unit/v1/test_upload_utils.py index 89b91e8fb6..9b0b0b1852 100644 --- a/glance/tests/unit/v1/test_upload_utils.py +++ b/glance/tests/unit/v1/test_upload_utils.py @@ -17,6 +17,7 @@ from contextlib import contextmanager import mock from mock import patch +import glance_store import webob.exc from glance.api.v1 import upload_utils @@ -210,12 +211,12 @@ class TestUploadUtils(base.StoreClearingUnitTest): def test_upload_data_to_store_storage_full(self): self._test_upload_data_to_store_exception_with_notify( - exception.StorageFull, + glance_store.StorageFull, webob.exc.HTTPRequestEntityTooLarge) def test_upload_data_to_store_storage_write_denied(self): self._test_upload_data_to_store_exception_with_notify( - exception.StorageWriteDenied, + glance_store.StorageWriteDenied, webob.exc.HTTPServiceUnavailable) def test_upload_data_to_store_size_limit_exceeded(self): diff --git a/glance/tests/unit/v2/test_image_data_resource.py b/glance/tests/unit/v2/test_image_data_resource.py index 26d6cb9929..5a08c06c79 100644 --- a/glance/tests/unit/v2/test_image_data_resource.py +++ b/glance/tests/unit/v2/test_image_data_resource.py @@ -16,6 +16,7 @@ import mock import uuid +import glance_store import six import webob @@ -228,7 +229,7 @@ class TestImagesController(base.StoreClearingUnitTest): def test_upload_storage_full(self): request = unit_test_utils.get_fake_request() image = FakeImage() - image.set_data = Raise(exception.StorageFull) + image.set_data = Raise(glance_store.StorageFull) self.image_repo.result = image self.assertRaises(webob.exc.HTTPRequestEntityTooLarge, self.controller.upload, @@ -268,7 +269,7 @@ class TestImagesController(base.StoreClearingUnitTest): def test_upload_storage_write_denied(self): request = unit_test_utils.get_fake_request(user=unit_test_utils.USER3) image = FakeImage() - image.set_data = Raise(exception.StorageWriteDenied) + image.set_data = Raise(glance_store.StorageWriteDenied) self.image_repo.result = image self.assertRaises(webob.exc.HTTPServiceUnavailable, self.controller.upload, @@ -328,7 +329,7 @@ class TestImagesController(base.StoreClearingUnitTest): def test_restore_image_when_upload_failed(self): request = unit_test_utils.get_fake_request() image = FakeImage('fake') - image.set_data = Raise(exception.StorageWriteDenied) + image.set_data = Raise(glance_store.StorageWriteDenied) self.image_repo.result = image self.assertRaises(webob.exc.HTTPServiceUnavailable, self.controller.upload, diff --git a/glance/tests/unit/v2/test_image_members_resource.py b/glance/tests/unit/v2/test_image_members_resource.py index 8d6985ff86..67ce312ddf 100644 --- a/glance/tests/unit/v2/test_image_members_resource.py +++ b/glance/tests/unit/v2/test_image_members_resource.py @@ -15,6 +15,7 @@ import datetime +import glance_store from oslo.config import cfg import webob @@ -101,7 +102,7 @@ class TestImageMembersController(test_utils.BaseTestCase): self.policy, self.notifier, self.store) - glance.store.create_stores() + glance_store.create_stores() def _create_images(self): self.db.reset() diff --git a/glance/tests/unit/v2/test_images_resource.py b/glance/tests/unit/v2/test_images_resource.py index 06a8d33780..9ecbb1501c 100644 --- a/glance/tests/unit/v2/test_images_resource.py +++ b/glance/tests/unit/v2/test_images_resource.py @@ -14,8 +14,10 @@ # under the License. import datetime +import os import uuid +import glance_store as store from oslo.config import cfg import six import testtools @@ -25,7 +27,6 @@ import glance.api.v2.images from glance.common import exception from glance.openstack.common import jsonutils import glance.schema -import glance.store from glance.tests.unit import base import glance.tests.unit.utils as unit_test_utils import glance.tests.utils as test_utils @@ -126,7 +127,7 @@ class TestImagesController(base.IsolatedUnitTest): self.notifier, self.store) self.controller.gateway.store_utils = self.store_utils - glance.store.create_stores() + store.create_stores() def _create_images(self): self.db.reset() @@ -1270,7 +1271,7 @@ class TestImagesController(base.IsolatedUnitTest): another_request, created_image.image_id, changes) def test_update_replace_locations(self): - self.stubs.Set(glance.store, 'get_size_from_backend', + self.stubs.Set(store, 'get_size_from_backend', unit_test_utils.fake_get_size_from_backend) request = unit_test_utils.get_fake_request() changes = [{'op': 'replace', 'path': ['locations'], 'value': []}] @@ -1537,7 +1538,7 @@ class TestImagesController(base.IsolatedUnitTest): as long as the image has fewer than the limited number of image locations after the transaction. """ - self.stubs.Set(glance.store, 'get_size_from_backend', + self.stubs.Set(store, 'get_size_from_backend', unit_test_utils.fake_get_size_from_backend) self.config(show_multiple_locations=True) request = unit_test_utils.get_fake_request() @@ -1599,7 +1600,7 @@ class TestImagesController(base.IsolatedUnitTest): self.controller.update, request, UUID1, changes) def test_update_remove_location(self): - self.stubs.Set(glance.store, 'get_size_from_backend', + self.stubs.Set(store, 'get_size_from_backend', unit_test_utils.fake_get_size_from_backend) request = unit_test_utils.get_fake_request() @@ -1729,7 +1730,9 @@ class TestImagesController(base.IsolatedUnitTest): Ensure status of queued image is updated (LP bug #1048851) to 'deleted' when delayed_delete isenabled """ - self.config(delayed_delete=True) + scrubber_dir = os.path.join(self.test_dir, 'scrubber') + self.config(delayed_delete=True, scrubber_datadir=scrubber_dir) + request = unit_test_utils.get_fake_request(is_admin=True) image = self.db.image_create(request.context, {'status': 'queued'}) image_id = image['id'] @@ -1756,7 +1759,8 @@ class TestImagesController(base.IsolatedUnitTest): self.assertNotIn('%s/%s' % (BASE_URI, UUID1), self.store.data) def test_delayed_delete(self): - self.config(delayed_delete=True) + scrubber_dir = os.path.join(self.test_dir, 'scrubber') + self.config(delayed_delete=True, scrubber_datadir=scrubber_dir) request = unit_test_utils.get_fake_request() self.assertIn('%s/%s' % (BASE_URI, UUID1), self.store.data) diff --git a/glance/tests/unit/v2/test_registry_api.py b/glance/tests/unit/v2/test_registry_api.py index 5a89eb931f..70a6a3249b 100644 --- a/glance/tests/unit/v2/test_registry_api.py +++ b/glance/tests/unit/v2/test_registry_api.py @@ -32,7 +32,6 @@ from glance.openstack.common import jsonutils from glance.openstack.common import timeutils from glance.registry.api import v2 as rserver -import glance.store.filesystem from glance.tests.unit import base from glance.tests import utils as test_utils diff --git a/requirements.txt b/requirements.txt index ca97170ec9..92d7e560ec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -51,3 +51,6 @@ oslo.messaging>=1.4.0.0a3 retrying>=1.2.2 # Apache-2.0 osprofiler>=0.3.0 + +# Glance Store +glance_store>=0.1.1