From 611b4c91ab6a5b06b5358b1f792e3b78268b9a53 Mon Sep 17 00:00:00 2001 From: Emilien Macchi Date: Sun, 27 Oct 2019 21:26:15 +0100 Subject: [PATCH] Fix backward compatibility for old config startup files In I1cf8923a698d0f6e0b1e00a7985f363a83e914c4, we changed the format for container-startup-config and now have one JSON file per container, per step. It'll make it easier to operate containers one by one, instead of in one big JSON per step. However this change wasn't 100% backward compatible, and this patch aims to fix it. This patch does: 1) Support a directory of configs without container name The --file argument can now be a directory where the container configuration file is located. Example: paunch debug (...) --file /var/lib/tripleo-config/container-startup-config/step_1 All configs will be returned. 2) Support a directory of config and a container name Example: paunch debug (...) --container haproxy --file /var/lib/tripleo-config/container-startup-config/step_1 Only the container config will be returned. 3) Support the old format file without container name If the user specifies: --file /var/lib/tripleo-config/hashed-container-startup-config-step_1.json It'll return all container configs for the JSON files in: /var/lib/tripleo-config/container-startup-config/step_1/ (directory) 4) Support the old format file with container name If the user specifies: --container haproxy --file /var/lib/tripleo-config/hashed-container-startup-config-step_1.json It'll return the hashed container config file: /var/lib/tripleo-config/container-startup-config/step_1/hashed-haproxy.json 5) Add support for running paunch with a file + new format The new format would be: --container haproxy --file /var/lib/tripleo-config/container-startup-config/step_1/hashed-haproxy.json The container config would be returned. Note: if no name is specified, it'll try to guess the name based on the file name. It'll remove "hashed-' from it in case it was an hashed file. This patch should resolve all backward compatibility issues so Paunch can be used with both the new and old format. Closes-Bug: #1850050 Change-Id: I917679da22fa09614e73053654df6ce181cf98fe (cherry picked from commit 68ecaf90611718e1fb1ac77e7eb199036ff1c769) --- paunch/cmd.py | 17 +++---- paunch/tests/test_utils_common.py | 74 ++++++++++++++++++++++++++++++- paunch/utils/common.py | 70 +++++++++++++++++++++++++++++ 3 files changed, 152 insertions(+), 9 deletions(-) diff --git a/paunch/cmd.py b/paunch/cmd.py index d088255..df749bc 100644 --- a/paunch/cmd.py +++ b/paunch/cmd.py @@ -16,7 +16,6 @@ import collections from cliff import command from cliff import lister import json -import yaml import paunch @@ -33,7 +32,8 @@ class Apply(command.Command): '--file', metavar='', required=True, - help=('YAML or JSON file containing configuration data'), + help=('YAML or JSON file or directory containing configuration ' + 'data'), ) parser.add_argument( '--label', @@ -89,8 +89,8 @@ class Apply(command.Command): k, v = l.split(('='), 1) labels[k] = v - with open(parsed_args.file, 'r') as f: - config = yaml.safe_load(f) + config_path = parsed_args.file + config = utils.common.load_config(config_path) stdout, stderr, rc = paunch.apply( parsed_args.config_id, @@ -198,7 +198,8 @@ class Debug(command.Command): '--file', metavar='', required=True, - help=('YAML or JSON file containing configuration data') + help=('YAML or JSON file or directory containing configuration ' + 'data'), ) parser.add_argument( '--label', @@ -286,10 +287,10 @@ class Debug(command.Command): k, v = l.split(('='), 1) labels[k] = v - with open(parsed_args.file, 'r') as f: - config = yaml.safe_load(f) - container_name = parsed_args.container_name + config_path = parsed_args.file + config = utils.common.load_config(config_path, container_name) + cconfig = {} cconfig[container_name] = config[container_name] diff --git a/paunch/tests/test_utils_common.py b/paunch/tests/test_utils_common.py index 65f18b6..9210f1c 100644 --- a/paunch/tests/test_utils_common.py +++ b/paunch/tests/test_utils_common.py @@ -19,7 +19,7 @@ from paunch.tests import base from paunch.utils import common -class TestUtilsCommon(base.TestCase): +class TestUtilsCommonCpu(base.TestCase): @mock.patch("psutil.Process.cpu_affinity", return_value=[0, 1, 2, 3]) def test_get_cpus_allowed_list(self, mock_cpu): @@ -32,3 +32,75 @@ class TestUtilsCommon(base.TestCase): expected_list = '0-3' actual_list = common.get_all_cpus() self.assertEqual(actual_list, expected_list) + + +class TestUtilsCommonConfig(base.TestCase): + + def setUp(self): + super(TestUtilsCommonConfig, self).setUp() + self.config_content = "{'image': 'docker.io/haproxy'}" + self.open_func = 'paunch.utils.common.open' + self.expected_config = {'haproxy': {'image': 'docker.io/haproxy'}} + self.container = 'haproxy' + self.old_config_file = '/var/lib/tripleo-config/' + \ + 'hashed-container-startup-config-step_1.json' + self.old_config_content = "{'haproxy': {'image': 'docker.io/haproxy'}}" + + @mock.patch('os.path.isdir') + def test_load_config_dir_with_name(self, mock_isdir): + mock_isdir.return_value = True + mock_open = mock.mock_open(read_data=self.config_content) + with mock.patch(self.open_func, mock_open): + self.assertEqual( + self.expected_config, + common.load_config('/config_dir', self.container)) + + @mock.patch('os.path.isdir') + @mock.patch('glob.glob') + def test_load_config_dir_without_name(self, mock_glob, mock_isdir): + mock_isdir.return_value = True + mock_glob.return_value = ['hashed-haproxy.json'] + mock_open = mock.mock_open(read_data=self.config_content) + with mock.patch(self.open_func, mock_open): + self.assertEqual( + self.expected_config, + common.load_config('/config_dir')) + + @mock.patch('os.path.isdir') + def test_load_config_file_with_name(self, mock_isdir): + mock_isdir.return_value = False + mock_open = mock.mock_open(read_data=self.config_content) + with mock.patch(self.open_func, mock_open): + self.assertEqual( + self.expected_config, + common.load_config('/config_dir/haproxy.json', self.container)) + + @mock.patch('os.path.isdir') + def test_load_config_file_without_name(self, mock_isdir): + mock_isdir.return_value = False + mock_open = mock.mock_open(read_data=self.config_content) + with mock.patch(self.open_func, mock_open): + self.assertEqual( + self.expected_config, + common.load_config('/config_dir/haproxy.json')) + + @mock.patch('os.path.isdir') + def test_load_config_file_backward_compat_with_name(self, mock_isdir): + mock_isdir.return_value = False + mock_open = mock.mock_open(read_data=self.old_config_content) + with mock.patch(self.open_func, mock_open): + self.assertEqual( + self.expected_config, + common.load_config(self.old_config_file, self.container)) + + @mock.patch('os.path.isdir') + @mock.patch('glob.glob') + def test_load_config_file_backward_compat_without_name(self, mock_glob, + mock_isdir): + mock_isdir.return_value = False + mock_glob.return_value = ['hashed-haproxy.json'] + mock_open = mock.mock_open(read_data=self.old_config_content) + with mock.patch(self.open_func, mock_open): + self.assertEqual( + self.expected_config, + common.load_config(self.old_config_file)) diff --git a/paunch/utils/common.py b/paunch/utils/common.py index 5f68028..6805a10 100644 --- a/paunch/utils/common.py +++ b/paunch/utils/common.py @@ -13,10 +13,13 @@ # License for the specific language governing permissions and limitations # under the License. +import glob import logging import os import psutil +import re import sys +import yaml from paunch import constants from paunch import utils @@ -81,3 +84,70 @@ def get_all_cpus(**args): :return: Value computed by psutil, e.g. '0-3' """ return "0-" + str(psutil.cpu_count() - 1) + + +def load_config(config, name=None): + container_config = {} + if os.path.isdir(config): + # When the user gives a config directory and specify a container name, + # we return the container config for that specific container. + if name: + cf = 'hashed-' + name + '.json' + with open(os.path.join(config, cf), 'r') as f: + container_config[name] = {} + container_config[name].update(yaml.safe_load(f)) + # When the user gives a config directory and without container name, + # we return all container configs in that directory. + else: + config_files = glob.glob(os.path.join(config, 'hashed-*.json')) + for cf in config_files: + with open(os.path.join(config, cf), 'r') as f: + name = os.path.basename(os.path.splitext( + cf.replace('hashed-', ''))[0]) + container_config[name] = {} + container_config[name].update(yaml.safe_load(f)) + else: + # Backward compatibility so our users can still use the old path, + # paunch will recognize it and find the right container config. + old_format = '/var/lib/tripleo-config/hashed-container-startup-config' + if config.startswith(old_format): + step = re.search('/var/lib/tripleo-config/' + 'hashed-container-startup-config-step' + '_(.+).json', config).group(1) + # If a name is specified, we return the container config for that + # specific container. + if name: + new_path = os.path.join( + '/var/lib/tripleo-config/container_startup_config', + 'step_' + step, 'hashed-' + name + '.json') + with open(new_path, 'r') as f: + c_config = yaml.safe_load(f) + container_config[name] = {} + container_config[name].update(c_config[name]) + # When no name is specified, we return all container configs in + # the file. + else: + new_path = os.path.join( + '/var/lib/tripleo-config/container_startup_config', + 'step_' + step) + config_files = glob.glob(os.path.join(new_path, + 'hashed-*.json')) + for cf in config_files: + with open(os.path.join(new_path, cf), 'r') as f: + name = os.path.basename(os.path.splitext( + cf.replace('hashed-', ''))[0]) + c_config = yaml.safe_load(f) + container_config[name] = {} + container_config[name].update(c_config[name]) + # When the user gives a file path, that isn't the old format, + # we consider it's the new format so the file name is the container + # name. + else: + if not name: + # No name was given, we'll guess it with file name + name = os.path.basename(os.path.splitext( + config.replace('hashed-', ''))[0]) + with open(os.path.join(config), 'r') as f: + container_config[name] = {} + container_config[name].update(yaml.safe_load(f)) + return container_config