From e54deefaff07fcc921f8c4115a7cd6b36cceff48 Mon Sep 17 00:00:00 2001 From: Rick Harris Date: Fri, 22 Jul 2011 00:53:43 -0500 Subject: [PATCH] Adding reaper script --- bin/glance | 33 +++++++++++-- bin/glance-reaper | 70 ++++++++++++++++++++++++++++ etc/glance-pruner.conf | 10 ---- etc/glance-reaper.conf | 28 +++++++++++ glance/api/cached_images.py | 4 ++ glance/api/middleware/image_cache.py | 3 +- glance/client.py | 7 +++ glance/image_cache/__init__.py | 26 +++++++++++ glance/image_cache/pruner.py | 24 ---------- glance/image_cache/reaper.py | 54 +++++++++++++++++++++ setup.py | 1 + 11 files changed, 220 insertions(+), 40 deletions(-) create mode 100755 bin/glance-reaper create mode 100644 etc/glance-reaper.conf create mode 100644 glance/image_cache/reaper.py diff --git a/bin/glance b/bin/glance index 5d0171d51e..4d23320d64 100755 --- a/bin/glance +++ b/bin/glance @@ -632,6 +632,25 @@ Removes all images from the cache""" print "done" +@catch_error('reap invalid images') +def cache_reap_invalid(options, args): + """ +%(prog)s cache-reap-invalid [options] + +Reaps any invalid images that were left for +debugging purposes""" + if not options.force and \ + not user_confirm("Reap all invalid cached images?", default=False): + print 'Not reaping any invalid cached images.' + return FAILURE + + client = get_client(options) + client.reap_invalid_cached_images() + + if options.verbose: + print "done" + + @catch_error('prefetch the specified cached image') def cache_prefetch(options, args): """ @@ -783,10 +802,11 @@ def lookup_command(parser, command_name): 'cache-index': cache_index, 'cache-invalid': cache_invalid, 'cache-incomplete': cache_incomplete, + 'cache-prefetching': cache_prefetching, + 'cache-prefetch': cache_prefetch, 'cache-purge': cache_purge, 'cache-clear': cache_clear, - 'cache-prefetch': cache_prefetch, - 'cache-prefetching': cache_prefetching} + 'cache-reap-invalid': cache_reap_invalid} commands = {} for command_set in (BASE_COMMANDS, IMAGE_COMMANDS, CACHE_COMMANDS): @@ -856,13 +876,16 @@ Cache Commands: cache-incomplete List images currently being fetched + cache-prefetching List images that are being prefetched + + cache-prefetch Pre-fetch an image or list of images into the cache + cache-purge Purges an image from the cache cache-clear Removes all images from the cache - cache-prefetch Pre-fetch an image or list of images into the cache - - cache-prefetching List images that are being prefetched + cache-reap-invalid Reaps any invalid images that were left for + debugging purposes """ oparser = optparse.OptionParser(version='%%prog %s' diff --git a/bin/glance-reaper b/bin/glance-reaper new file mode 100755 index 0000000000..67e9bf3ccb --- /dev/null +++ b/bin/glance-reaper @@ -0,0 +1,70 @@ +#!/usr/bin/env python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Glance Image Cache Invalid Cache Entry Reaper + +This is meant to be run as a periodic task from cron. + +If something goes wrong while we're caching an image (for example the fetch +times out, or an exception is raised), we create an 'invalid' entry. These +entires are left around for debugging purposes. However, after some period of +time, we want to cleans these up, aka reap them. +""" + +import optparse +import os +import sys + +# If ../glance/__init__.py exists, add ../ to Python search path, so that +# it will override what happens to be installed in /usr/(local/)lib/python... +possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, + os.pardir)) +if os.path.exists(os.path.join(possible_topdir, 'glance', '__init__.py')): + sys.path.insert(0, possible_topdir) + +from glance import version +from glance.common import config +from glance.common import wsgi + + +def create_options(parser): + """ + Sets up the CLI and config-file options that may be + parsed and program commands. + + :param parser: The option parser + """ + config.add_common_options(parser) + config.add_log_options(parser) + + +if __name__ == '__main__': + oparser = optparse.OptionParser(version='%%prog %s' + % version.version_string()) + create_options(oparser) + (options, args) = config.parse_options(oparser) + + try: + conf, app = config.load_paste_app('glance-reaper', options, args) + app.run() + except RuntimeError, e: + sys.exit("ERROR: %s" % e) diff --git a/etc/glance-pruner.conf b/etc/glance-pruner.conf index e898155bb9..d402af9df8 100644 --- a/etc/glance-pruner.conf +++ b/etc/glance-pruner.conf @@ -27,15 +27,5 @@ image_cache_percent_extra_to_free = 0.20 # Make sure this is also set in glance-api.conf image_cache_datadir = /var/lib/glance/image-cache/ -# image_cache_invalid_entry_grace_period - seconds -# -# If an exception is raised as we're writing to the cache, the cache-entry is -# deemed invalid and moved to /invalid so that it can be -# inspected for debugging purposes. -# -# This is number of seconds to leave these invalid images around before they -# are elibible to be pruned. -image_cache_invalid_entry_grace_period = 3600 - [app:glance-pruner] paste.app_factory = glance.image_cache.pruner:app_factory diff --git a/etc/glance-reaper.conf b/etc/glance-reaper.conf new file mode 100644 index 0000000000..d90bcd193a --- /dev/null +++ b/etc/glance-reaper.conf @@ -0,0 +1,28 @@ +[DEFAULT] +# Show more verbose log output (sets INFO log level output) +verbose = True + +# Show debugging output in logs (sets DEBUG log level output) +debug = False + +log_file = /var/log/glance/reaper.log + +# Whether to enable the caching of image data +image_cache_enabled = False + +# Directory that the Image Cache writes data to +# Make sure this is also set in glance-api.conf +image_cache_datadir = /var/lib/glance/image-cache/ + +# image_cache_invalid_entry_grace_period - seconds +# +# If an exception is raised as we're writing to the cache, the cache-entry is +# deemed invalid and moved to /invalid so that it can be +# inspected for debugging purposes. +# +# This is number of seconds to leave these invalid images around before they +# are elibible to be pruned. +image_cache_invalid_entry_grace_period = 3600 + +[app:glance-reaper] +paste.app_factory = glance.image_cache.reaper:app_factory diff --git a/glance/api/cached_images.py b/glance/api/cached_images.py index 6a4afbca51..e6e0c0386b 100644 --- a/glance/api/cached_images.py +++ b/glance/api/cached_images.py @@ -60,6 +60,10 @@ class Controller(api.BaseController): def clear(self, req): self.cache.clear() + def reap(self, req): + """Reaps any invalid cached images""" + self.cache.reap_invalid() + def update(self, req, id): """PUT /cached_images/1 is used to prefetch an image into the cache""" image_meta = self.get_image_meta_or_404(req, id) diff --git a/glance/api/middleware/image_cache.py b/glance/api/middleware/image_cache.py index 6e89decaa1..d15115359f 100644 --- a/glance/api/middleware/image_cache.py +++ b/glance/api/middleware/image_cache.py @@ -34,7 +34,8 @@ class ImageCacheFilter(wsgi.Middleware): map = app.map resource = cached_images.create_resource(options) map.resource("cached_image", "cached_images", - controller=resource) + controller=resource, + collection={'reap': 'POST'}) map.connect("/cached_images", controller=resource, diff --git a/glance/client.py b/glance/client.py index b9d5a3dd2b..bf2354166e 100644 --- a/glance/client.py +++ b/glance/client.py @@ -226,6 +226,13 @@ class V1Client(base_client.BaseClient): self.do_request("DELETE", "/cached_images") return True + def reap_invalid_cached_images(self): + """ + Reaps any invalid cached images + """ + self.do_request("POST", "/cached_images/reap") + return True + def prefetch_cache_image(self, image_id): """ Pre-fetch a specified image from the cache diff --git a/glance/image_cache/__init__.py b/glance/image_cache/__init__.py index 46573a39fb..86a2d051df 100644 --- a/glance/image_cache/__init__.py +++ b/glance/image_cache/__init__.py @@ -24,6 +24,7 @@ import itertools import logging import os import sys +import time from glance.common import config from glance.common import exception @@ -404,3 +405,28 @@ class ImageCache(object): path = entry['path'] entry['hits'] = utils.get_xattr(path, 'hits', default='UNKNOWN') yield entry + + def reap_invalid(self, grace=None): + """Remove any invalid cache entries + + :param grace: Number of seconds to keep an invalid entry around for + debugging purposes. If None, then delete immediately. + """ + now = time.time() + + reaped = 0 + for path in self.get_all_regular_files(self.invalid_path): + mtime = os.path.getmtime(path) + age = now - mtime + if not grace: + logger.debug("No grace period, reaping '%(path)s' immediately" + % locals()) + self._delete_file(path) + reaped += 1 + elif age > grace: + logger.debug("Cache entry '%(path)s' exceeds grace period, " + "(%(age)i s > %(grace)i s)" % locals()) + self._delete_file(path) + reaped += 1 + + logger.info("Reaped %(reaped)s invalid cache entries" % locals()) diff --git a/glance/image_cache/pruner.py b/glance/image_cache/pruner.py index 69be93db28..56fa8dcf67 100644 --- a/glance/image_cache/pruner.py +++ b/glance/image_cache/pruner.py @@ -47,12 +47,6 @@ class Pruner(object): self.options, 'image_cache_percent_extra_to_free', type='float', default=0.05) - @property - def image_cache_invalid_entry_grace_period(self): - return config.get_option( - self.options, 'image_cache_invalid_entry_grace_period', - type='int', default=3600) - def run(self): if not self.cache.enabled: logger.debug( @@ -60,24 +54,6 @@ class Pruner(object): return self.prune_cache() - self.prune_invalid_cache_entries() - - def prune_invalid_cache_entries(self): - """Prune invalid cache entries that are older than the grace period""" - grace = self.image_cache_invalid_entry_grace_period - now = time.time() - - pruned = 0 - for path in self.cache.get_all_regular_files(self.cache.invalid_path): - mtime = os.path.getmtime(path) - age = now - mtime - if age > grace: - logger.debug("Cache entry '%(path)s' exceeds grace period, " - "(%(age)i s > %(grace)i s)" % locals()) - self.cache._delete_file(path) - pruned += 1 - - logger.info("Pruned %(pruned)s invalid cache entries" % locals()) def prune_cache(self): """Prune the cache using an LRU strategy""" diff --git a/glance/image_cache/reaper.py b/glance/image_cache/reaper.py new file mode 100644 index 0000000000..24e7560ef9 --- /dev/null +++ b/glance/image_cache/reaper.py @@ -0,0 +1,54 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Reaps any invalid cache entries that exceed the grace period +""" +import logging + +from glance.common import config +from glance.image_cache import ImageCache + + +logger = logging.getLogger('glance.image_cache.reaper') + + +class Reaper(object): + def __init__(self, options): + self.options = options + self.cache = ImageCache(options) + + @property + def image_cache_invalid_entry_grace_period(self): + return config.get_option( + self.options, 'image_cache_invalid_entry_grace_period', + type='int', default=3600) + + def run(self): + if not self.cache.enabled: + logger.debug( + "Image caching is not enabled, going back to sleep...") + return + + grace = self.image_cache_invalid_entry_grace_period + self.cache.reap_invalid(grace=grace) + + +def app_factory(global_config, **local_conf): + conf = global_config.copy() + conf.update(local_conf) + return Reaper(conf) diff --git a/setup.py b/setup.py index 8f0af490f6..21253afe87 100644 --- a/setup.py +++ b/setup.py @@ -91,5 +91,6 @@ setup( 'bin/glance-manage', 'bin/glance-prefetcher', 'bin/glance-pruner', + 'bin/glance-reaper', 'bin/glance-registry', 'bin/glance-upload'])