From f7a359cd0edde7be287340699c2d588ebb53098e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Harald=20Jens=C3=A5s?= Date: Thu, 27 Sep 2018 15:48:12 +0200 Subject: [PATCH] Merge new params - nic-config templates Utility script to merge new parameters into existing nic-config templates. Uses process-templates.py rendered 'single-nic-vlans' templates as reference and appends any parameters that is not already present in the existing NIC template. New NIC template parameters were introduced in: https://review.openstack.org/#/c/580236/ When upgrading the existing NIC templates have to have these new parameters merged. Change-Id: I474e57878212d2cb7c2b392a5fdf4e449f783a66 --- tools/merge-new-params-nic-config-script.py | 267 ++++++++++++++++++++ 1 file changed, 267 insertions(+) create mode 100755 tools/merge-new-params-nic-config-script.py diff --git a/tools/merge-new-params-nic-config-script.py b/tools/merge-new-params-nic-config-script.py new file mode 100755 index 0000000000..910e0e88a1 --- /dev/null +++ b/tools/merge-new-params-nic-config-script.py @@ -0,0 +1,267 @@ +#!/usr/bin/env python +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import argparse +import collections +import datetime +import os +import re +import shutil +import subprocess +import sys +import yaml + +from tempfile import mkdtemp + +DEFAULT_THT_DIR = '/usr/share/openstack-tripleo-heat-templates' +NIC_CONFIG_REFERENCE = 'single-nic-vlans' + + +def parse_opts(argv): + parser = argparse.ArgumentParser( + description='Merge new NIC config template parameters into ' + 'existing NIC config template.') + parser.add_argument('-r', '--roles-data', metavar='ROLES_DATA', + help="Relative path to the roles_data.yaml file.", + default=('%s/roles_data.yaml') % DEFAULT_THT_DIR) + parser.add_argument('-n', '--network-data', metavar='NETWORK_DATA', + help="Relative path to the network_data.yaml file.", + default=('%s/network_data.yaml') % DEFAULT_THT_DIR) + parser.add_argument('--role-name', metavar='ROLE-NAME', + help="Name of the role the NIC config is used for.", + default='Controller') + parser.add_argument('-t', '--template', metavar='TEMPLATE_FILE', + help=("Existing NIC config template to merge " + "parameter too."), + required=True, + ) + parser.add_argument('--tht-dir', metavar='THT_DIR', + help=("Path to tripleo-heat-templates (THT) " + "directory"), + default=DEFAULT_THT_DIR) + parser.add_argument('--discard-comments', metavar='DISCARD_COMMENTS', + help="Discard comments from the template. (The " + "scripts functions to keep YAML file comments in " + "place, does not work in all scenarios.)", + default=False) + + opts = parser.parse_args(argv[1:]) + + return opts + + +# FIXME: This duplicates code from tools/yaml-nic-config-2-script.py, we should +# refactor to share the common code +def to_commented_yaml(filename): + """ Convert comments into 'comments: ...' YAML """ + + out_str = '' + last_non_comment_spaces = '' + with open(filename, 'r') as f: + comment_count = 0 + for line in f: + # skip blank line + if line.isspace(): + continue + char_count = 0 + spaces = '' + for char in line: + char_count += 1 + if char == ' ': + spaces += ' ' + next + elif char == '#': + comment_count += 1 + comment = line[char_count:-1] + last_non_comment_spaces = spaces + out_str += "%scomment%i_%i: '%s'\n" % ( + last_non_comment_spaces, comment_count, len(spaces), + comment) + break + else: + last_non_comment_spaces = spaces + out_str += line + + # inline comments check + m = re.match(".*:.*#(.*)", line) + if m: + comment_count += 1 + out_str += "%s inline_comment%i: '%s'\n" % ( + last_non_comment_spaces, comment_count, m.group(1)) + break + + with open(filename, 'w') as f: + f.write(out_str) + + return out_str + + +# FIXME: This duplicates code from tools/yaml-nic-config-2-script.py, we should +# refactor to share the common code +def to_normal_yaml(filename): + """ Convert back to normal #commented YAML""" + + with open(filename, 'r') as f: + data = f.read() + + out_str = '' + next_line_break = False + for line in data.split('\n'): + # get_input not supported by run-os-net-config.sh script + line = line.replace('get_input: ', '') + # Normal comments + m = re.match(" +comment[0-9]+_([0-9]+): '(.*)'.*", line) + # Inline comments + i = re.match(" +inline_comment[0-9]+: '(.*)'.*", line) + if m: + if next_line_break: + out_str += '\n' + next_line_break = False + for x in range(0, int(m.group(1))): + out_str += " " + out_str += "#%s\n" % m.group(2) + elif i: + out_str += " #%s\n" % i.group(1) + next_line_break = False + else: + if next_line_break: + out_str += '\n' + out_str += line + next_line_break = True + + if next_line_break: + out_str += '\n' + + with open(filename, 'w') as f: + f.write(out_str) + + return out_str + + +# FIXME: Some of this duplicates code from build_endpoint_map.py, we should +# refactor to share the common code +class TemplateDumper(yaml.SafeDumper): + def represent_ordered_dict(self, data): + return self.represent_dict(data.items()) + + def description_presenter(self, data): + if not len(data) > 80: + return self.represent_scalar('tag:yaml.org,2002:str', data) + return self.represent_scalar('tag:yaml.org,2002:str', data, style='>') + + +# FIXME: This duplicates code from tools/yaml-nic-config-2-script.py, we should +# refactor to share the common code +# We load mappings into OrderedDict to preserve their order +class TemplateLoader(yaml.SafeLoader): + def construct_mapping(self, node): + self.flatten_mapping(node) + return collections.OrderedDict(self.construct_pairs(node)) + + +if sys.version_info.major >= 3: + TemplateDumper.add_representer(str, TemplateDumper.description_presenter) + TemplateDumper.add_representer(bytes, + TemplateDumper.description_presenter) +else: + TemplateDumper.add_representer(str, TemplateDumper.description_presenter) + TemplateDumper.add_representer(unicode, + TemplateDumper.description_presenter) + +TemplateDumper.add_representer(collections.OrderedDict, + TemplateDumper.represent_ordered_dict) +TemplateLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, + TemplateLoader.construct_mapping) + + +# FIXME: This duplicates code from tools/yaml-nic-config-2-script.py, we should +# refactor to share the common code +def write_template(template, filename=None): + with open(filename, 'w') as f: + yaml.dump(template, f, TemplateDumper, width=120, + default_flow_style=False) + + +def process_templates_and_get_reference_parameters(): + temp_dir = mkdtemp(dir='/tmp') + executable = OPTS.tht_dir + '/tools/process-templates.py' + cmd = [executable, + '--roles-data ' + OPTS.roles_data, + '--base_path ' + OPTS.tht_dir, + '--network-data ' + OPTS.network_data, + '--output-dir ' + temp_dir] + child = subprocess.Popen(' '.join(cmd), shell=True, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, err = child.communicate() + if not child.returncode == 0: + raise RuntimeError('Error processing templates: %s' % err) + + # If deprecated_nic_config_names is set for role the deprecated name must + # be used when loading the reference file. + with open(OPTS.roles_data) as roles_data_file: + roles_data = yaml.safe_load(roles_data_file) + nic_config_name = next((x.get('deprecated_nic_config_name', + OPTS.role_name.lower() + '.yaml') for x in + roles_data if x['name'] == OPTS.role_name)) + + refernce_file = '/'.join([temp_dir, 'network/config', NIC_CONFIG_REFERENCE, + nic_config_name]) + with open(refernce_file) as reference: + reference_template = yaml.safe_load(reference) + reference_params = reference_template['parameters'] + shutil.rmtree(temp_dir) + + return reference_params + + +def validate_template(): + if not os.path.exists(OPTS.template): + raise RuntimeError('Template not provided.') + if not os.path.isfile(OPTS.template): + raise RuntimeError('Template %s is not a file.') + pass + + +def backup_template(): + extension = datetime.datetime.now().strftime('%Y%m%d%H%M%S') + backup_filename = os.path.realpath(OPTS.template) + '.' + extension + if os.path.exists(backup_filename): + raise RuntimeError('Backupe file: %s already exists. Aborting!' + % backup_filename) + shutil.copyfile(OPTS.template, backup_filename) + print('The original template was saved as: %s' % backup_filename) + + +def merge_from_processed(reference_params): + template = yaml.load(open(OPTS.template).read(), Loader=TemplateLoader) + for param in reference_params: + if param not in template['parameters']: + template['parameters'][param] = reference_params[param] + + write_template(template, filename=OPTS.template) + print('The update template was saved as: %s' % OPTS.template) + + +OPTS = parse_opts(sys.argv) +validate_template() +backup_template() +if not OPTS.discard_comments: + # Convert comments '# ...' into 'comments: ...' YAML so that the info + # is not lost when loading the data. + to_commented_yaml(OPTS.template) +reference_params = process_templates_and_get_reference_parameters() +merge_from_processed(reference_params) +if not OPTS.discard_comments: + # Convert previously converted comments, 'comments: ...' YAML back to + # normal #commented YAML + to_normal_yaml(OPTS.template)