openstack
/
heat
Code Issues Proposed changes

Enable caching for property constraints 

The patch adds support of oslo.caching in heat and enables caching
when heat requests OS services and validates custom constraints.
So it allows to increase performance of validation (cause we
don't need to call requests for the same values) and
handle the cases when stack have a lot of instances (and
glance fails because heat is ddosing it).
So the logic introduced in this patch is the following:
- heat request value from validation cache
- if value is found and it is not expired then we return it
otherwise we need request client plugin about validation
- if value is valid then store result in cache
otherwise do not store it.

The patch also introduces list of configurations that needs to be
specified when using caching.
The configuration for caching is modular. It consists from
global toggle for caching (disabled by default because in-memory
cache is leaking and memcached is not installed by default) and
local toggles for specific piece of functionality
(constraints cache). Local toggle is enabled by default but it
won't be activated without global toggle(enabled=True in
heat.conf).

DocImpact
Implements: blueprint constraint-validation-cache
Change-Id: I0a71c52a5c003f0fc6be1762647ffe0146a3b387
This commit is contained in:
kairat_kushaev 2015-07-15 16:11:44 +03:00 committed by Kairat Kushaev
parent bd7895c555
commit 130ada151a
6 changed files with 113 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
heat/common/cache.py Normal file
View File

@ -0,0 +1,71 @@
#
# Copyright 2015 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_cache import core
from oslo_config import cfg
from heat.common.i18n import _
"""The module contains the code related to integration between oslo.cache
module and heat."""
def register_cache_configurations(conf):
"""Register all configurations required for oslo.cache
The procedure registers all configurations required for oslo.cache.
It should be called before configuring of cache region
:param conf: instance of heat configuration
:return updated heat configuration
"""
# register global configurations for caching in heat
core.configure(conf)
# register heat specific configurations
constraint_cache_group = cfg.OptGroup('constraint_validation_cache')
constraint_cache_opts = [
cfg.IntOpt('expiration_time', default=60,
help=_(
'TTL, in seconds, for any cached item in the '
'dogpile.cache region used for caching of validation '
'constraints.')),
cfg.BoolOpt("caching", default=True,
help=_(
'Toggle to enable/disable caching when Orchestration '
'Engine validates property constraints of stack.'
'During property validation with constraints '
'Orchestration Engine caches requests to other '
'OpenStack services. Please note that the global '
'toggle for oslo.cache(enabled=True in [cache] group) '
'must be enabled to use this feature.'))
]
conf.register_group(constraint_cache_group)
conf.register_opts(constraint_cache_opts, group=constraint_cache_group)
return conf
# variable that stores an initialized cache region for heat
_REGION = None
def get_cache_region():
global _REGION
if not _REGION:
_REGION = core.configure_cache_region(
conf=register_cache_configurations(cfg.CONF),
region=core.create_region())
return _REGION

0
heat/common/cache/__init__.py vendored
View File

0
heat/common/cache/backends/__init__.py vendored
View File

48
heat/common/cache/backends/noop.py vendored
View File

@ -1,48 +0,0 @@
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from dogpile.cache import api
NO_VALUE = api.NO_VALUE
class NoopCacheBackend(api.CacheBackend):
"""A no op backend as a default caching backend.
The no op backend is provided as the default caching backend for heat
to ensure that ``dogpile.cache.memory`` is not used in any real-world
circumstances unintentionally. ``dogpile.cache.memory`` does not have a
mechanism to cleanup it's internal dict and therefore could cause run-away
memory utilization.
"""
def __init__(self, *args):
return
def get(self, key):
return NO_VALUE
def get_multi(self, keys):
return [NO_VALUE for x in keys]
def set(self, key, value):
return
def set_multi(self, mapping):
return
def delete(self, key):
return
def delete_multi(self, keys):
return

48
heat/engine/constraints.py
View File

@ -16,13 +16,22 @@ import numbers
import re
import warnings
from oslo_cache import core
from oslo_config import cfg
from oslo_utils import strutils
import six
from heat.common import cache
from heat.common import exception
from heat.common.i18n import _
from heat.engine import resources
# decorator that allows to cache the value
# of the function based on input arguments
MEMOIZE = core.get_memoization_decorator(conf=cfg.CONF,
region=cache.get_cache_region(),
group="constraint_validation_cache")
class Schema(collections.Mapping):
"""
@ -582,10 +591,35 @@ class BaseCustomConstraint(object):
"value": value, "message": self._error_message}
def validate(self, value, context):
try:
self.validate_with_client(context.clients, value)
except self.expected_exceptions as e:
self._error_message = str(e)
return False
else:
return True
@MEMOIZE
def check_cache_or_validate_value(class_name, value_to_validate):
"""Check if validation result stored in cache or validate value.
The function checks that value was validated and validation
result stored in cache. If not then it executes validation and
stores the result of validation in cache.
If caching is disable it requests for validation each time.
:param class_name: name of custom constraint class, it will be
used in cache key to distinguish values
:param value_to_validate: value that need to be validated
:return: True if value is valid otherwise False
"""
try:
self.validate_with_client(context.clients, value_to_validate)
except self.expected_exceptions as e:
self._error_message = str(e)
return False
else:
return True
validation_result = check_cache_or_validate_value(
self.__class__.__name__, value)
# if validation failed we should not store it in cache
# cause validation will be fixed soon (by admin or other guy)
# and we don't need to require user wait for expiration time
if not validation_result:
check_cache_or_validate_value.invalidate(self.__class__.__name__,
value)
return validation_result

1
requirements.txt
View File

@ -14,6 +14,7 @@ keystonemiddleware>=2.0.0
kombu>=3.0.7
lxml>=2.3
netaddr>=0.7.12
oslo.cache>=0.3.0 # Apache-2.0
oslo.config>=1.11.0 # Apache-2.0
oslo.concurrency>=2.1.0 # Apache-2.0
oslo.context>=0.2.0 # Apache-2.0