340 lines
12 KiB
Python
340 lines
12 KiB
Python
# Copyright 2011 OpenStack Foundation
|
|
# All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
"""
|
|
LRU Cache for Image Data
|
|
"""
|
|
|
|
import hashlib
|
|
|
|
from oslo_config import cfg
|
|
from oslo_utils import excutils
|
|
from oslo_utils import importutils
|
|
from oslo_utils import units
|
|
|
|
from glance.common import exception
|
|
from glance.common import utils
|
|
from glance import i18n
|
|
import glance.openstack.common.log as logging
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
_ = i18n._
|
|
_LE = i18n._LE
|
|
_LI = i18n._LI
|
|
_LW = i18n._LW
|
|
|
|
image_cache_opts = [
|
|
cfg.StrOpt('image_cache_driver', default='sqlite',
|
|
help=_('The driver to use for image cache management.')),
|
|
cfg.IntOpt('image_cache_max_size', default=10 * units.Gi, # 10 GB
|
|
help=_('The maximum size in bytes that the cache can use.')),
|
|
cfg.IntOpt('image_cache_stall_time', default=86400, # 24 hours
|
|
help=_('The amount of time to let an image remain in the '
|
|
'cache without being accessed.')),
|
|
cfg.StrOpt('image_cache_dir',
|
|
help=_('Base directory that the Image Cache uses.')),
|
|
]
|
|
|
|
CONF = cfg.CONF
|
|
CONF.register_opts(image_cache_opts)
|
|
|
|
|
|
class ImageCache(object):
|
|
|
|
"""Provides an LRU cache for image data."""
|
|
|
|
def __init__(self):
|
|
self.init_driver()
|
|
|
|
def init_driver(self):
|
|
"""
|
|
Create the driver for the cache
|
|
"""
|
|
driver_name = CONF.image_cache_driver
|
|
driver_module = (__name__ + '.drivers.' + driver_name + '.Driver')
|
|
try:
|
|
self.driver_class = importutils.import_class(driver_module)
|
|
LOG.info(_LI("Image cache loaded driver '%s'.") %
|
|
driver_name)
|
|
except ImportError as import_err:
|
|
LOG.warn(_LW("Image cache driver "
|
|
"'%(driver_name)s' failed to load. "
|
|
"Got error: '%(import_err)s."),
|
|
{'driver_name': driver_name,
|
|
'import_err': import_err})
|
|
|
|
driver_module = __name__ + '.drivers.sqlite.Driver'
|
|
LOG.info(_LI("Defaulting to SQLite driver."))
|
|
self.driver_class = importutils.import_class(driver_module)
|
|
self.configure_driver()
|
|
|
|
def configure_driver(self):
|
|
"""
|
|
Configure the driver for the cache and, if it fails to configure,
|
|
fall back to using the SQLite driver which has no odd dependencies
|
|
"""
|
|
try:
|
|
self.driver = self.driver_class()
|
|
self.driver.configure()
|
|
except exception.BadDriverConfiguration as config_err:
|
|
driver_module = self.driver_class.__module__
|
|
LOG.warn(_LW("Image cache driver "
|
|
"'%(driver_module)s' failed to configure. "
|
|
"Got error: '%(config_err)s"),
|
|
{'driver_module': driver_module,
|
|
'config_err': config_err})
|
|
LOG.info(_LI("Defaulting to SQLite driver."))
|
|
default_module = __name__ + '.drivers.sqlite.Driver'
|
|
self.driver_class = importutils.import_class(default_module)
|
|
self.driver = self.driver_class()
|
|
self.driver.configure()
|
|
|
|
def is_cached(self, image_id):
|
|
"""
|
|
Returns True if the image with the supplied ID has its image
|
|
file cached.
|
|
|
|
:param image_id: Image ID
|
|
"""
|
|
return self.driver.is_cached(image_id)
|
|
|
|
def is_queued(self, image_id):
|
|
"""
|
|
Returns True if the image identifier is in our cache queue.
|
|
|
|
:param image_id: Image ID
|
|
"""
|
|
return self.driver.is_queued(image_id)
|
|
|
|
def get_cache_size(self):
|
|
"""
|
|
Returns the total size in bytes of the image cache.
|
|
"""
|
|
return self.driver.get_cache_size()
|
|
|
|
def get_hit_count(self, image_id):
|
|
"""
|
|
Return the number of hits that an image has
|
|
|
|
:param image_id: Opaque image identifier
|
|
"""
|
|
return self.driver.get_hit_count(image_id)
|
|
|
|
def get_cached_images(self):
|
|
"""
|
|
Returns a list of records about cached images.
|
|
"""
|
|
return self.driver.get_cached_images()
|
|
|
|
def delete_all_cached_images(self):
|
|
"""
|
|
Removes all cached image files and any attributes about the images
|
|
and returns the number of cached image files that were deleted.
|
|
"""
|
|
return self.driver.delete_all_cached_images()
|
|
|
|
def delete_cached_image(self, image_id):
|
|
"""
|
|
Removes a specific cached image file and any attributes about the image
|
|
|
|
:param image_id: Image ID
|
|
"""
|
|
self.driver.delete_cached_image(image_id)
|
|
|
|
def delete_all_queued_images(self):
|
|
"""
|
|
Removes all queued image files and any attributes about the images
|
|
and returns the number of queued image files that were deleted.
|
|
"""
|
|
return self.driver.delete_all_queued_images()
|
|
|
|
def delete_queued_image(self, image_id):
|
|
"""
|
|
Removes a specific queued image file and any attributes about the image
|
|
|
|
:param image_id: Image ID
|
|
"""
|
|
self.driver.delete_queued_image(image_id)
|
|
|
|
def prune(self):
|
|
"""
|
|
Removes all cached image files above the cache's maximum
|
|
size. Returns a tuple containing the total number of cached
|
|
files removed and the total size of all pruned image files.
|
|
"""
|
|
max_size = CONF.image_cache_max_size
|
|
current_size = self.driver.get_cache_size()
|
|
if max_size > current_size:
|
|
LOG.debug("Image cache has free space, skipping prune...")
|
|
return (0, 0)
|
|
|
|
overage = current_size - max_size
|
|
LOG.debug("Image cache currently %(overage)d bytes over max "
|
|
"size. Starting prune to max size of %(max_size)d ",
|
|
{'overage': overage, 'max_size': max_size})
|
|
|
|
total_bytes_pruned = 0
|
|
total_files_pruned = 0
|
|
entry = self.driver.get_least_recently_accessed()
|
|
while entry and current_size > max_size:
|
|
image_id, size = entry
|
|
LOG.debug("Pruning '%(image_id)s' to free %(size)d bytes",
|
|
{'image_id': image_id, 'size': size})
|
|
self.driver.delete_cached_image(image_id)
|
|
total_bytes_pruned = total_bytes_pruned + size
|
|
total_files_pruned = total_files_pruned + 1
|
|
current_size = current_size - size
|
|
entry = self.driver.get_least_recently_accessed()
|
|
|
|
LOG.debug("Pruning finished pruning. "
|
|
"Pruned %(total_files_pruned)d and "
|
|
"%(total_bytes_pruned)d.",
|
|
{'total_files_pruned': total_files_pruned,
|
|
'total_bytes_pruned': total_bytes_pruned})
|
|
return total_files_pruned, total_bytes_pruned
|
|
|
|
def clean(self, stall_time=None):
|
|
"""
|
|
Cleans up any invalid or incomplete cached images. The cache driver
|
|
decides what that means...
|
|
"""
|
|
self.driver.clean(stall_time)
|
|
|
|
def queue_image(self, image_id):
|
|
"""
|
|
This adds a image to be cache to the queue.
|
|
|
|
If the image already exists in the queue or has already been
|
|
cached, we return False, True otherwise
|
|
|
|
:param image_id: Image ID
|
|
"""
|
|
return self.driver.queue_image(image_id)
|
|
|
|
def get_caching_iter(self, image_id, image_checksum, image_iter):
|
|
"""
|
|
Returns an iterator that caches the contents of an image
|
|
while the image contents are read through the supplied
|
|
iterator.
|
|
|
|
:param image_id: Image ID
|
|
:param image_checksum: checksum expected to be generated while
|
|
iterating over image data
|
|
:param image_iter: Iterator that will read image contents
|
|
"""
|
|
if not self.driver.is_cacheable(image_id):
|
|
return image_iter
|
|
|
|
LOG.debug("Tee'ing image '%s' into cache", image_id)
|
|
|
|
return self.cache_tee_iter(image_id, image_iter, image_checksum)
|
|
|
|
def cache_tee_iter(self, image_id, image_iter, image_checksum):
|
|
try:
|
|
current_checksum = hashlib.md5()
|
|
|
|
with self.driver.open_for_write(image_id) as cache_file:
|
|
for chunk in image_iter:
|
|
try:
|
|
cache_file.write(chunk)
|
|
finally:
|
|
current_checksum.update(chunk)
|
|
yield chunk
|
|
cache_file.flush()
|
|
|
|
if (image_checksum and
|
|
image_checksum != current_checksum.hexdigest()):
|
|
msg = _("Checksum verification failed. Aborted "
|
|
"caching of image '%s'.") % image_id
|
|
raise exception.GlanceException(msg)
|
|
|
|
except exception.GlanceException as e:
|
|
with excutils.save_and_reraise_exception():
|
|
# image_iter has given us bad, (size_checked_iter has found a
|
|
# bad length), or corrupt data (checksum is wrong).
|
|
LOG.exception(utils.exception_to_str(e))
|
|
except Exception as e:
|
|
LOG.exception(_LE("Exception encountered while tee'ing "
|
|
"image '%(image_id)s' into cache: %(error)s. "
|
|
"Continuing with response.") %
|
|
{'image_id': image_id,
|
|
'error': utils.exception_to_str(e)})
|
|
|
|
# If no checksum provided continue responding even if
|
|
# caching failed.
|
|
for chunk in image_iter:
|
|
yield chunk
|
|
|
|
def cache_image_iter(self, image_id, image_iter, image_checksum=None):
|
|
"""
|
|
Cache an image with supplied iterator.
|
|
|
|
:param image_id: Image ID
|
|
:param image_file: Iterator retrieving image chunks
|
|
:param image_checksum: Checksum of image
|
|
|
|
:retval True if image file was cached, False otherwise
|
|
"""
|
|
if not self.driver.is_cacheable(image_id):
|
|
return False
|
|
|
|
for chunk in self.get_caching_iter(image_id, image_checksum,
|
|
image_iter):
|
|
pass
|
|
return True
|
|
|
|
def cache_image_file(self, image_id, image_file):
|
|
"""
|
|
Cache an image file.
|
|
|
|
:param image_id: Image ID
|
|
:param image_file: Image file to cache
|
|
|
|
:retval True if image file was cached, False otherwise
|
|
"""
|
|
CHUNKSIZE = 64 * units.Mi
|
|
|
|
return self.cache_image_iter(image_id,
|
|
utils.chunkiter(image_file, CHUNKSIZE))
|
|
|
|
def open_for_read(self, image_id):
|
|
"""
|
|
Open and yield file for reading the image file for an image
|
|
with supplied identifier.
|
|
|
|
:note Upon successful reading of the image file, the image's
|
|
hit count will be incremented.
|
|
|
|
:param image_id: Image ID
|
|
"""
|
|
return self.driver.open_for_read(image_id)
|
|
|
|
def get_image_size(self, image_id):
|
|
"""
|
|
Return the size of the image file for an image with supplied
|
|
identifier.
|
|
|
|
:param image_id: Image ID
|
|
"""
|
|
return self.driver.get_image_size(image_id)
|
|
|
|
def get_queued_images(self):
|
|
"""
|
|
Returns a list of image IDs that are in the queue. The
|
|
list should be sorted by the time the image ID was inserted
|
|
into the queue.
|
|
"""
|
|
return self.driver.get_queued_images()
|