From 73f30b728656f617db0d8c3f29b0b1bd078b34c9 Mon Sep 17 00:00:00 2001 From: Jiri Stransky Date: Fri, 7 Oct 2016 18:29:46 +0200 Subject: [PATCH] Fix get_file in out-of-tree templates We already have special processing in place for out-of-tree environment files and templates, but it didn't handle `get_file` (or `type`) links. This commit adds that handling. Heatclient automatically uses absolute `file:///` links when processing the external environment files and other files referenced from them via `get_file` or `type`. Because we upload all our environment files and templates to Swift, such links don't work. We need to use relative links in `get_file` and `type`. Change-Id: I009f75cbc6278a0a2ff75e93e1ed44f2c4893783 Closes-Bug: #1631426 --- tripleoclient/tests/test_utils.py | 67 ++++++++++++++++++++++++++++ tripleoclient/utils.py | 66 +++++++++++++++++++++++++++ tripleoclient/v1/overcloud_deploy.py | 20 +++++---- 3 files changed, 144 insertions(+), 9 deletions(-) diff --git a/tripleoclient/tests/test_utils.py b/tripleoclient/tests/test_utils.py index 5d861fb35..f3af42c50 100644 --- a/tripleoclient/tests/test_utils.py +++ b/tripleoclient/tests/test_utils.py @@ -19,6 +19,7 @@ import mock import os.path import tempfile from unittest import TestCase +import yaml from tripleoclient import exceptions from tripleoclient.tests.v1.utils import ( @@ -718,3 +719,69 @@ class TestAssignVerifyProfiles(TestCase): self.nodes[:] = [self._get_fake_node(profile=None)] self.flavors = {'baremetal': (FakeFlavor('baremetal', None), 1)} self._test(0, 0) + + +class TestReplaceLinks(TestCase): + + def setUp(self): + super(TestReplaceLinks, self).setUp() + self.link_replacement = { + 'file:///home/stack/test.sh': + 'user-files/home/stack/test.sh', + 'file:///usr/share/extra-templates/my.yml': + 'user-files/usr/share/extra-templates/my.yml', + } + + def test_replace_links(self): + source = ( + 'description: my template\n' + 'heat_template_version: "2014-10-16"\n' + 'resources:\n' + ' test_config:\n' + ' properties:\n' + ' config: {get_file: "file:///home/stack/test.sh"}\n' + ' type: OS::Heat::SoftwareConfig\n' + ) + expected = ( + 'description: my template\n' + 'heat_template_version: "2014-10-16"\n' + 'resources:\n' + ' test_config:\n' + ' properties:\n' + ' config: {get_file: user-files/home/stack/test.sh}\n' + ' type: OS::Heat::SoftwareConfig\n' + ) + + # the yaml->string dumps aren't always character-precise, so + # we need to parse them into dicts for comparison + expected_dict = yaml.safe_load(expected) + result_dict = yaml.safe_load(utils.replace_links_in_template_contents( + source, self.link_replacement)) + self.assertEqual(expected_dict, result_dict) + + def test_replace_links_not_template(self): + # valid JSON/YAML, but doesn't have heat_template_version + source = '{"get_file": "file:///home/stack/test.sh"}' + self.assertEqual( + source, + utils.replace_links_in_template_contents( + source, self.link_replacement)) + + def test_replace_links_not_yaml(self): + # invalid JSON/YAML -- curly brace left open + source = '{"invalid JSON"' + self.assertEqual( + source, + utils.replace_links_in_template_contents( + source, self.link_replacement)) + + def test_relative_link_replacement(self): + current_dir = 'user-files/home/stack' + expected = { + 'file:///home/stack/test.sh': + 'test.sh', + 'file:///usr/share/extra-templates/my.yml': + '../../usr/share/extra-templates/my.yml', + } + self.assertEqual(expected, utils.relative_link_replacement( + self.link_replacement, current_dir)) diff --git a/tripleoclient/utils.py b/tripleoclient/utils.py index a06160fe1..e884ceb2c 100644 --- a/tripleoclient/utils.py +++ b/tripleoclient/utils.py @@ -900,3 +900,69 @@ def parse_env_file(env_file, file_type=None): nodes_config = nodes_config['nodes'] return nodes_config + + +def replace_links_in_template_contents(contents, link_replacement): + """Replace get_file and type file links in Heat template contents + + If the string contents passed in is a Heat template, scan the + template for 'get_file' and 'type' occurences, and replace the + file paths according to link_replacement dict. (Key/value in + link_replacement are from/to, respectively.) + + If the string contents don't look like a Heat template, return the + contents unmodified. + """ + + template = {} + try: + template = yaml.safe_load(contents) + except yaml.YAMLError: + return contents + + if not (isinstance(template, dict) and + template.get('heat_template_version')): + return contents + + template = replace_links_in_template(template, link_replacement) + + return yaml.safe_dump(template) + + +def replace_links_in_template(template_part, link_replacement): + """Replace get_file and type file links in a Heat template + + Scan the template for 'get_file' and 'type' occurences, and + replace the file paths according to link_replacement + dict. (Key/value in link_replacement are from/to, respectively.) + """ + + def replaced_dict_value(key, value): + if ((key == 'get_file' or key == 'type') and + isinstance(value, six.string_types)): + return link_replacement.get(value, value) + else: + return replace_links_in_template(value, link_replacement) + + def replaced_list_value(value): + return replace_links_in_template(value, link_replacement) + + if isinstance(template_part, dict): + return {k: replaced_dict_value(k, v) + for k, v in six.iteritems(template_part)} + elif isinstance(template_part, list): + return map(replaced_list_value, template_part) + else: + return template_part + + +def relative_link_replacement(link_replacement, current_dir): + """Generate a relative version of link_replacement dictionary. + + Get a link_replacement dictionary (where key/value are from/to + respectively), and make the values in that dictionary relative + paths with respect to current_dir. + """ + + return {k: os.path.relpath(v, current_dir) + for k, v in six.iteritems(link_replacement)} diff --git a/tripleoclient/v1/overcloud_deploy.py b/tripleoclient/v1/overcloud_deploy.py index bbbcda705..8ca8f7f65 100644 --- a/tripleoclient/v1/overcloud_deploy.py +++ b/tripleoclient/v1/overcloud_deploy.py @@ -16,7 +16,6 @@ from __future__ import print_function import argparse import glob -import hashlib import logging import os import os.path @@ -360,7 +359,8 @@ class DeployOvercloud(command.Command): file_relocation = {} file_prefix = "file://" - for fullpath, contents in files_dict.items(): + # select files files for relocation & upload + for fullpath in files_dict.keys(): if not fullpath.startswith(file_prefix): continue @@ -371,13 +371,15 @@ class DeployOvercloud(command.Command): # This should already be uploaded. continue - filename = os.path.basename(path) - checksum = hashlib.md5() - checksum.update(path) - digest = checksum.hexdigest() - swift_path = "user-files/{}-{}".format(digest, filename) - swift_client.put_object(container_name, swift_path, contents) - file_relocation[fullpath] = swift_path + file_relocation[fullpath] = "user-files/{}".format(path[1:]) + + # make sure links within files point to new locations, and upload them + for orig_path, reloc_path in file_relocation.items(): + link_replacement = utils.relative_link_replacement( + file_relocation, os.path.dirname(reloc_path)) + contents = utils.replace_links_in_template_contents( + files_dict[orig_path], link_replacement) + swift_client.put_object(container_name, reloc_path, contents) return file_relocation