diff --git a/deckhand/engine/layering.py b/deckhand/engine/layering.py index f253fc23..d89278cf 100644 --- a/deckhand/engine/layering.py +++ b/deckhand/engine/layering.py @@ -659,7 +659,7 @@ class DocumentLayering(object): pass doc.data = rendered_data.data self.secrets_substitution.update_substitution_sources( - doc.schema, doc.name, rendered_data.data) + doc.meta, rendered_data.data) self._documents_by_index[doc.meta] = rendered_data else: LOG.debug( @@ -686,7 +686,7 @@ class DocumentLayering(object): doc.data = rendered_data.data if not doc.has_replacement: self.secrets_substitution.update_substitution_sources( - doc.schema, doc.name, rendered_data.data) + doc.meta, rendered_data.data) self._documents_by_index[doc.meta] = rendered_data # Otherwise, retrieve the encrypted data for the document if its # data has been encrypted so that future references use the actual @@ -697,7 +697,7 @@ class DocumentLayering(object): if not doc.is_abstract: doc.data = encrypted_data self.secrets_substitution.update_substitution_sources( - doc.schema, doc.name, encrypted_data) + doc.meta, encrypted_data) self._documents_by_index[doc.meta] = encrypted_data # NOTE: Since the child-replacement is always prioritized, before diff --git a/deckhand/engine/secrets_manager.py b/deckhand/engine/secrets_manager.py index 71b6ed3a..688a80be 100644 --- a/deckhand/engine/secrets_manager.py +++ b/deckhand/engine/secrets_manager.py @@ -344,11 +344,36 @@ class SecretsSubstitution(object): yield document - def update_substitution_sources(self, schema, name, data): + def update_substitution_sources(self, meta, data): + """Update substitution sources with rendered data so that future + layering and substitution sources reference the latest rendered data + rather than stale data. + + :param meta: Tuple of (schema, layer, name). + :type meta: tuple + :param data: Dictionary of just-rendered document data that belongs + to the document uniquely identified by ``meta``. + :type data: dict + :returns None + """ + schema, layer, name = meta + if (schema, name) not in self._substitution_sources: return + # Substitution sources only use schema/name which doesn't uniquely + # identify replacement documents. The check below ensures that the + # exact document is selected. Note that Deckhand prioritizes the + # child-replacement to be rendered immediately after the + # parent-replacement document, meaning that the child-replacement + # document will be the one who correctly updates the substitution + # sources below (which don't include parent-replacement documents). + # Afterward, all other documents that reference the parent-replacement + # should get the correct data. substitution_src = self._substitution_sources[(schema, name)] + if substitution_src.meta != meta: + return + if isinstance(data, dict) and isinstance(substitution_src.data, dict): substitution_src.data.update(data) else: diff --git a/deckhand/tests/functional/gabbits/replacement/multi-layer-replacement.yaml b/deckhand/tests/functional/gabbits/replacement/multi-layer-replacement.yaml new file mode 100644 index 00000000..8cd970aa --- /dev/null +++ b/deckhand/tests/functional/gabbits/replacement/multi-layer-replacement.yaml @@ -0,0 +1,129 @@ +# Tests success path for advanced replacement scenario, where +# parent-replacement (type layer) layers with global document, after which +# the parent-replacement is replaced by the child-replacement (site layer). +# +# 1. Purges existing data to ensure test isolation. +# 2. Adds initial documents with replacement scenario described above. +# 3. Verifies correctly layered, substituted and replaced data. + +defaults: + request_headers: + content-type: application/x-yaml + response_headers: + content-type: application/x-yaml + verbose: true + +tests: + - name: purge + desc: Begin testing from known state. + DELETE: /api/v1.0/revisions + status: 204 + response_headers: null + + - name: initialize + desc: |- + Create initial documents to validate following scenario: + + * Global document called nova-global + * Region document called nova (layers with nova-global) + * Site document (replaces nova) + + PUT: /api/v1.0/buckets/mop/documents + status: 200 + data: |- + --- + schema: deckhand/LayeringPolicy/v1 + metadata: + schema: metadata/Control/v1 + name: layering-policy + data: + layerOrder: + - global + - type + - site + --- + schema: armada/Chart/v1 + metadata: + schema: metadata/Document/v1 + name: nova-global + labels: + name: nova-global + component: nova + layeringDefinition: + abstract: false + layer: global + data: + values: + pod: + replicas: + server: 16 + --- + schema: armada/Chart/v1 + metadata: + schema: metadata/Document/v1 + name: nova + labels: + name: nova-5ec + component: nova + layeringDefinition: + abstract: false + layer: type + parentSelector: + name: nova-global + actions: + - method: merge + path: . + data: {} + --- + schema: armada/Chart/v1 + metadata: + schema: metadata/Document/v1 + replacement: true + name: nova + layeringDefinition: + abstract: false + layer: site + parentSelector: + name: nova-5ec + actions: + - method: merge + path: . + data: + values: + pod: + replicas: + api_metadata: 16 + placement: 2 + osapi: 16 + conductor: 16 + consoleauth: 2 + scheduler: 2 + novncproxy: 2 + + - name: verify_multi_layer_replacement_document + desc: | + Tests success path for advanced replacement scenario, where + parent-replacement document (type layer) layers with document from + global layer, after which it is replaced by the child-replacement + document (site layer). + GET: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/rendered-documents + query_parameters: + schema: armada/Chart/v1 + metadata.name: nova + status: 200 + response_multidoc_jsonpaths: + $.`len`: 1 + $.[*].metadata.name: nova + $.[*].metadata.layeringDefinition.layer: site + $.[*].data: + values: + pod: + replicas: + api_metadata: 16 + placement: 2 + osapi: 16 + conductor: 16 + consoleauth: 2 + scheduler: 2 + novncproxy: 2 + server: 16 diff --git a/deckhand/tests/unit/engine/test_document_layering_and_replacement.py b/deckhand/tests/unit/engine/test_document_layering_and_replacement.py index 2e72457e..a500078b 100644 --- a/deckhand/tests/unit/engine/test_document_layering_and_replacement.py +++ b/deckhand/tests/unit/engine/test_document_layering_and_replacement.py @@ -251,3 +251,115 @@ data: self._test_layering( documents, site_expected=site_expected, region_expected=None) + + def test_multi_layer_replacement(self): + """Validate the following scenario: + + * Global document called nova-global + * Region document called nova (layers with nova-global) + * Site document (replaces nova) + """ + self.documents = list(yaml.safe_load_all(""" +--- +schema: deckhand/LayeringPolicy/v1 +metadata: + schema: metadata/Control/v1 + name: layering-policy +data: + layerOrder: + - global + - region + - site +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: nova-global + labels: + name: nova-global + component: nova + layeringDefinition: + abstract: false + layer: global +data: + values: + pod: + replicas: + server: 16 +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: nova + labels: + name: nova-5ec + component: nova + layeringDefinition: + abstract: false + layer: region + parentSelector: + name: nova-global + actions: + - method: merge + path: . +data: {} +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + replacement: true + name: nova + layeringDefinition: + abstract: false + layer: site + parentSelector: + name: nova-5ec + actions: + - method: merge + path: . +data: + values: + pod: + replicas: + api_metadata: 16 + placement: 2 + osapi: 16 + conductor: 16 + consoleauth: 2 + scheduler: 2 + novncproxy: 2 +""")) + + site_expected = [ + { + "values": { + "pod": { + "replicas": { + "api_metadata": 16, + "placement": 2, + "osapi": 16, + "conductor": 16, + "consoleauth": 2, + "scheduler": 2, + "novncproxy": 2, + "server": 16 + } + } + } + } + ] + global_expected = [ + { + "values": { + "pod": { + "replicas": { + "server": 16 + } + } + } + } + ] + self._test_layering(self.documents, + site_expected=site_expected, + region_expected=None, + global_expected=global_expected) diff --git a/tools/whitespace-linter.sh b/tools/whitespace-linter.sh index 3709cdf3..62ad0b60 100755 --- a/tools/whitespace-linter.sh +++ b/tools/whitespace-linter.sh @@ -7,6 +7,8 @@ RES=$(find . \ -not -path "*/releasenotes/build/*" \ -not -path "*/doc/build/*" \ -not -name "*.tgz" \ + -not -name "*.html" \ + -not -name "*.pyc" \ -type f -exec egrep -l " +$" {} \;) if [[ -n $RES ]]; then