Merge "Implement rendered documents caching"

This commit is contained in:
Zuul 2018-07-31 19:31:07 +00:00 committed by Gerrit Code Review
commit d932ad2e35
9 changed files with 206 additions and 11 deletions

View File

@ -39,6 +39,26 @@ barbican_opts = [
]
engine_group = cfg.OptGroup(
name='engine',
title='Engine Options',
help="Engine options for allowing behavior specific to Deckhand's engine "
"to be configured.")
engine_opts = [
# TODO(felipemonteiro): This is better off being removed because the same
# effect can be achieved through per-test gabbi fixtures that clean up
# the cache between tests.
cfg.BoolOpt('enable_cache', default=True,
help="Whether to enable the document rendering caching. Useful"
" for testing to avoid cross-test caching conflicts."),
cfg.IntOpt('cache_timeout', default=3600,
help="How long (in seconds) document rendering results should "
"remain cached in memory."),
]
jsonpath_group = cfg.OptGroup(
name='jsonpath',
title='JSONPath Options',
@ -47,8 +67,8 @@ jsonpath_group = cfg.OptGroup(
jsonpath_opts = [
cfg.IntOpt('cache_timeout', default=3600,
help="How long JSONPath lookup results should remain cached "
"in memory.")
help="How long (in seconds) JSONPath lookup results should "
"remain cached in memory.")
]
@ -65,6 +85,7 @@ default_opts = [
def register_opts(conf):
conf.register_group(barbican_group)
conf.register_opts(barbican_opts, group=barbican_group)
conf.register_opts(engine_opts, group=engine_group)
conf.register_opts(jsonpath_opts, group=jsonpath_group)
conf.register_opts(default_opts)
ks_loading.register_auth_conf_options(conf, group='keystone_authtoken')

View File

@ -27,8 +27,8 @@ from deckhand.control import base as api_base
from deckhand.control import common
from deckhand.control.views import document as document_view
from deckhand.db.sqlalchemy import api as db_api
from deckhand import engine
from deckhand.engine import document_validation
from deckhand.engine import layering
from deckhand.engine import secrets_manager
from deckhand import errors
from deckhand import policy
@ -119,13 +119,10 @@ class RenderedDocumentsResource(api_base.BaseResource):
documents = document_wrapper.DocumentDict.from_list(data)
encryption_sources = self._resolve_encrypted_data(documents)
try:
# NOTE(fmontei): `validate` is False because documents have already
# been pre-validated during ingestion. Documents are post-validated
# below, regardless.
document_layering = layering.DocumentLayering(
documents, encryption_sources=encryption_sources,
validate=False)
rendered_documents = document_layering.render()
rendered_documents = engine.render(
revision_id,
documents,
encryption_sources=encryption_sources)
except (errors.BarbicanClientException,
errors.BarbicanServerException,
errors.InvalidDocumentLayer,

View File

@ -0,0 +1,17 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from deckhand.engine.render import render
__all__ = ('render',)

49
deckhand/engine/cache.py Normal file
View File

@ -0,0 +1,49 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from beaker.cache import CacheManager
from beaker.util import parse_cache_config_options
from oslo_log import log as logging
from deckhand.conf import config
from deckhand.engine import layering
CONF = config.CONF
LOG = logging.getLogger(__name__)
_CACHE_OPTS = {
'cache.type': 'memory',
'expire': CONF.engine.cache_timeout,
}
_CACHE = CacheManager(**parse_cache_config_options(_CACHE_OPTS))
_DOCUMENT_RENDERING_CACHE = _CACHE.get_cache('rendered_documents_cache')
def lookup_by_revision_id(revision_id, documents, **kwargs):
"""Look up rendered documents by ``revision_id``."""
def do_render():
"""Perform document rendering for the revision."""
document_layering = layering.DocumentLayering(documents, **kwargs)
return document_layering.render()
if CONF.engine.enable_cache:
return _DOCUMENT_RENDERING_CACHE.get(key=revision_id,
createfunc=do_render)
else:
return do_render()
def invalidate():
_DOCUMENT_RENDERING_CACHE.clear()

View File

@ -405,7 +405,7 @@ class DocumentLayering(object):
contained in the destination document's data section to the
actual unecrypted data. If encrypting data with Barbican, the
reference will be a Barbican secret reference.
:type encryption_sources: List[dict]
:type encryption_sources: dict
:raises LayeringPolicyNotFound: If no LayeringPolicy was found among
list of ``documents``.

45
deckhand/engine/render.py Normal file
View File

@ -0,0 +1,45 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from deckhand.engine import cache
__all__ = ('render',)
def render(revision_id, documents, encryption_sources=None):
"""Render revision documents for ``revision_id`` using raw ``documents``.
:param revision_id: Key used for caching rendered documents by.
:type revision_id: int
:param documents: List of raw documents corresponding to ``revision_id``
to render.
:type documents: List[dict]
:param encryption_sources: A dictionary that maps the reference
contained in the destination document's data section to the
actual unecrypted data. If encrypting data with Barbican, the
reference will be a Barbican secret reference.
:type encryption_sources: dict
:returns: Rendered documents for ``revision_id``.
:rtype: List[dict]
"""
# NOTE(felipemonteiro): `validate` is False because documents have
# already been pre-validated during ingestion. Documents are
# post-validated below, regardless.
return cache.lookup_by_revision_id(
revision_id,
documents,
encryption_sources=encryption_sources,
validate=False)

View File

@ -12,6 +12,9 @@ policy_file = policy.yaml
[database]
connection = ${AIRSHIP_DECKHAND_DATABASE_URL}
[engine]
enable_cache = false
[keystone_authtoken]
# NOTE(fmontei): Values taken from clouds.yaml. Values only used for
# integration testing.

View File

@ -24,6 +24,7 @@ import testtools
from deckhand.conf import config # noqa: Calls register_opts(CONF)
from deckhand.db.sqlalchemy import api as db_api
from deckhand.engine import cache
from deckhand.tests.unit import fixtures as dh_fixtures
CONF = cfg.CONF
@ -41,6 +42,11 @@ class DeckhandTestCase(testtools.TestCase):
self.useFixture(dh_fixtures.ConfPatcher(
development_mode=True, group=None))
def tearDown(self):
# Clear the cache between tests.
cache.invalidate()
super(DeckhandTestCase, self).tearDown()
def override_config(self, name, override, group=None):
CONF.set_override(name, override, group)
self.addCleanup(CONF.clear_override, name, group)

View File

@ -0,0 +1,57 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import testtools
from deckhand.engine import cache
from deckhand import factories
from deckhand.tests.unit import base as test_base
class RenderedDocumentsCacheTest(test_base.DeckhandTestCase):
def test_lookup_by_revision_id_cache(self):
"""Validate ``lookup_by_revision_id`` caching works.
Passing in None in lieu of the actual documents proves that:
* if the payload is in the cache, then no error is thrown since the
cache is hit so no further processing is performed, where otherwise a
method would be called on `None`
* if the payload is not in the cache, then following logic above,
method is called on `None`, raising AttributeError
"""
document_factory = factories.DocumentFactory(1, [1])
documents = document_factory.gen_test({})
# Validate that caching the ref returns expected payload.
rendered_documents = cache.lookup_by_revision_id(1, documents)
self.assertIsInstance(rendered_documents, list)
# Validate that the cache actually works.
next_rendered_documents = cache.lookup_by_revision_id(1, None)
self.assertEqual(rendered_documents, next_rendered_documents)
# No documents passed in and revision ID 2 isn't cached - so expect
# this to blow up.
with testtools.ExpectedException(AttributeError):
cache.lookup_by_revision_id(2, None)
# Invalidate the cache and ensure the original data isn't there.
cache.invalidate()
# The cache won't be hit this time - expect AttributeError.
with testtools.ExpectedException(AttributeError):
cache.lookup_by_revision_id(1, None)