openstack
/
paunch
Code Issues Proposed changes

Fix backward compatibility for old config startup files 

In I1cf8923a698d0f6e0b1e00a7985f363a83e914c4, we changed the format for
container-startup-config and now have one JSON file per container, per
step. It'll make it easier to operate containers one by one, instead of
in one big JSON per step.

However this change wasn't 100% backward compatible, and this patch aims
to fix it.

This patch does:

1) Support a directory of configs without container name

The --file argument can now be a directory where the container
configuration file is located.

Example:
paunch debug (...) --file /var/lib/tripleo-config/container-startup-config/step_1

All configs will be returned.

2) Support a directory of config and a container name

Example:
paunch debug (...) --container haproxy --file /var/lib/tripleo-config/container-startup-config/step_1

Only the container config will be returned.

3) Support the old format file without container name

If the user specifies:
--file /var/lib/tripleo-config/hashed-container-startup-config-step_1.json

It'll return all container configs for the JSON files in:
/var/lib/tripleo-config/container-startup-config/step_1/ (directory)

4) Support the old format file with container name

If the user specifies:
--container haproxy --file /var/lib/tripleo-config/hashed-container-startup-config-step_1.json

It'll return the hashed container config file:
/var/lib/tripleo-config/container-startup-config/step_1/hashed-haproxy.json

5) Add support for running paunch with a file + new format

The new format would be:

--container haproxy --file /var/lib/tripleo-config/container-startup-config/step_1/hashed-haproxy.json

The container config would be returned.

Note: if no name is specified, it'll try to guess the name based on the
file name. It'll remove "hashed-' from it in case it was an hashed file.

This patch should resolve all backward compatibility issues so Paunch
can be used with both the new and old format.

Closes-Bug: #1850050
Change-Id: I917679da22fa09614e73053654df6ce181cf98fe
(cherry picked from commit 68ecaf9061)
This commit is contained in:
Emilien Macchi 2019-10-27 21:26:15 +01:00
parent e0566017a5
commit 611b4c91ab
3 changed files with 152 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
paunch/cmd.py
View File

@ -16,7 +16,6 @@ import collections
from cliff import command
from cliff import lister
import json
import yaml
import paunch
@ -33,7 +32,8 @@ class Apply(command.Command):
'--file',
metavar='<file>',
required=True,
help=('YAML or JSON file containing configuration data'),
help=('YAML or JSON file or directory containing configuration '
'data'),
)
parser.add_argument(
'--label',
@ -89,8 +89,8 @@ class Apply(command.Command):
k, v = l.split(('='), 1)
labels[k] = v
with open(parsed_args.file, 'r') as f:
config = yaml.safe_load(f)
config_path = parsed_args.file
config = utils.common.load_config(config_path)
stdout, stderr, rc = paunch.apply(
parsed_args.config_id,
@ -198,7 +198,8 @@ class Debug(command.Command):
'--file',
metavar='<file>',
required=True,
help=('YAML or JSON file containing configuration data')
help=('YAML or JSON file or directory containing configuration '
'data'),
)
parser.add_argument(
'--label',
@ -286,10 +287,10 @@ class Debug(command.Command):
k, v = l.split(('='), 1)
labels[k] = v
with open(parsed_args.file, 'r') as f:
config = yaml.safe_load(f)
container_name = parsed_args.container_name
config_path = parsed_args.file
config = utils.common.load_config(config_path, container_name)
cconfig = {}
cconfig[container_name] = config[container_name]

74
paunch/tests/test_utils_common.py
View File

@ -19,7 +19,7 @@ from paunch.tests import base
from paunch.utils import common
class TestUtilsCommon(base.TestCase):
class TestUtilsCommonCpu(base.TestCase):
@mock.patch("psutil.Process.cpu_affinity", return_value=[0, 1, 2, 3])
def test_get_cpus_allowed_list(self, mock_cpu):
@ -32,3 +32,75 @@ class TestUtilsCommon(base.TestCase):
expected_list = '0-3'
actual_list = common.get_all_cpus()
self.assertEqual(actual_list, expected_list)
class TestUtilsCommonConfig(base.TestCase):
def setUp(self):
super(TestUtilsCommonConfig, self).setUp()
self.config_content = "{'image': 'docker.io/haproxy'}"
self.open_func = 'paunch.utils.common.open'
self.expected_config = {'haproxy': {'image': 'docker.io/haproxy'}}
self.container = 'haproxy'
self.old_config_file = '/var/lib/tripleo-config/' + \
'hashed-container-startup-config-step_1.json'
self.old_config_content = "{'haproxy': {'image': 'docker.io/haproxy'}}"
@mock.patch('os.path.isdir')
def test_load_config_dir_with_name(self, mock_isdir):
mock_isdir.return_value = True
mock_open = mock.mock_open(read_data=self.config_content)
with mock.patch(self.open_func, mock_open):
self.assertEqual(
self.expected_config,
common.load_config('/config_dir', self.container))
@mock.patch('os.path.isdir')
@mock.patch('glob.glob')
def test_load_config_dir_without_name(self, mock_glob, mock_isdir):
mock_isdir.return_value = True
mock_glob.return_value = ['hashed-haproxy.json']
mock_open = mock.mock_open(read_data=self.config_content)
with mock.patch(self.open_func, mock_open):
self.assertEqual(
self.expected_config,
common.load_config('/config_dir'))
@mock.patch('os.path.isdir')
def test_load_config_file_with_name(self, mock_isdir):
mock_isdir.return_value = False
mock_open = mock.mock_open(read_data=self.config_content)
with mock.patch(self.open_func, mock_open):
self.assertEqual(
self.expected_config,
common.load_config('/config_dir/haproxy.json', self.container))
@mock.patch('os.path.isdir')
def test_load_config_file_without_name(self, mock_isdir):
mock_isdir.return_value = False
mock_open = mock.mock_open(read_data=self.config_content)
with mock.patch(self.open_func, mock_open):
self.assertEqual(
self.expected_config,
common.load_config('/config_dir/haproxy.json'))
@mock.patch('os.path.isdir')
def test_load_config_file_backward_compat_with_name(self, mock_isdir):
mock_isdir.return_value = False
mock_open = mock.mock_open(read_data=self.old_config_content)
with mock.patch(self.open_func, mock_open):
self.assertEqual(
self.expected_config,
common.load_config(self.old_config_file, self.container))
@mock.patch('os.path.isdir')
@mock.patch('glob.glob')
def test_load_config_file_backward_compat_without_name(self, mock_glob,
mock_isdir):
mock_isdir.return_value = False
mock_glob.return_value = ['hashed-haproxy.json']
mock_open = mock.mock_open(read_data=self.old_config_content)
with mock.patch(self.open_func, mock_open):
self.assertEqual(
self.expected_config,
common.load_config(self.old_config_file))

70
paunch/utils/common.py
View File

@ -13,10 +13,13 @@
# License for the specific language governing permissions and limitations
# under the License.
import glob
import logging
import os
import psutil
import re
import sys
import yaml
from paunch import constants
from paunch import utils
@ -81,3 +84,70 @@ def get_all_cpus(**args):
:return: Value computed by psutil, e.g. '0-3'
"""
return "0-" + str(psutil.cpu_count() - 1)
def load_config(config, name=None):
container_config = {}
if os.path.isdir(config):
# When the user gives a config directory and specify a container name,
# we return the container config for that specific container.
if name:
cf = 'hashed-' + name + '.json'
with open(os.path.join(config, cf), 'r') as f:
container_config[name] = {}
container_config[name].update(yaml.safe_load(f))
# When the user gives a config directory and without container name,
# we return all container configs in that directory.
else:
config_files = glob.glob(os.path.join(config, 'hashed-*.json'))
for cf in config_files:
with open(os.path.join(config, cf), 'r') as f:
name = os.path.basename(os.path.splitext(
cf.replace('hashed-', ''))[0])
container_config[name] = {}
container_config[name].update(yaml.safe_load(f))
else:
# Backward compatibility so our users can still use the old path,
# paunch will recognize it and find the right container config.
old_format = '/var/lib/tripleo-config/hashed-container-startup-config'
if config.startswith(old_format):
step = re.search('/var/lib/tripleo-config/'
'hashed-container-startup-config-step'
'_(.+).json', config).group(1)
# If a name is specified, we return the container config for that
# specific container.
if name:
new_path = os.path.join(
'/var/lib/tripleo-config/container_startup_config',
'step_' + step, 'hashed-' + name + '.json')
with open(new_path, 'r') as f:
c_config = yaml.safe_load(f)
container_config[name] = {}
container_config[name].update(c_config[name])
# When no name is specified, we return all container configs in
# the file.
else:
new_path = os.path.join(
'/var/lib/tripleo-config/container_startup_config',
'step_' + step)
config_files = glob.glob(os.path.join(new_path,
'hashed-*.json'))
for cf in config_files:
with open(os.path.join(new_path, cf), 'r') as f:
name = os.path.basename(os.path.splitext(
cf.replace('hashed-', ''))[0])
c_config = yaml.safe_load(f)
container_config[name] = {}
container_config[name].update(c_config[name])
# When the user gives a file path, that isn't the old format,
# we consider it's the new format so the file name is the container
# name.
else:
if not name:
# No name was given, we'll guess it with file name
name = os.path.basename(os.path.splitext(
config.replace('hashed-', ''))[0])
with open(os.path.join(config), 'r') as f:
container_config[name] = {}
container_config[name].update(yaml.safe_load(f))
return container_config