Filter container images by deployed services
This change adds the --service-environment-file/-e argument to the prepare command which will take heat environment files that specify which services are containerised. When specified, the resulting image entries will be filtered only by the containerised services discovered in the environment. -e is used as the short argument so that tools like quickstart can use the same variable to specify these environments for both the prepare command and the overcloud deploy command. Once this change merges, any calls to prepare in CI can be modified to reduce the number of images being transferred for 'upload' calls. For example, specifying -e environment/docker.yaml will reduce image transfers from 85 to 44. (cherry picked from commit9ff9b7c663
) and commitcd1eb1e111
) Change-Id: I35866fb24915ef80b86cc402015afb08cb155dcf Partial-Bug: #1710992 Partial-Bug: #1714270 Depends-On: #Ie8801b11921c46923b0f7c9aaba6bf524c464e82
This commit is contained in:
parent
e6b77c8848
commit
0bd00bba4a
|
@ -0,0 +1,5 @@
|
|||
---
|
||||
features:
|
||||
- The "openstack overcloud container image prepare" command can now filter
|
||||
the image list by the containerized services being deployed. This is done by
|
||||
specifying the heat environment files which enable containerized services.
|
|
@ -97,13 +97,23 @@ class TestContainerImagePrepare(TestPluginV1):
|
|||
)
|
||||
|
||||
@mock.patch('tripleo_common.image.kolla_builder.KollaImageBuilder')
|
||||
def test_container_image_prepare(self, mock_builder):
|
||||
@mock.patch('heatclient.common.template_utils.'
|
||||
'process_multiple_environments_and_files')
|
||||
def test_container_image_prepare(self, pmef, mock_builder):
|
||||
|
||||
temp = tempfile.mkdtemp()
|
||||
self.addCleanup(shutil.rmtree, temp)
|
||||
images_file = os.path.join(temp, 'overcloud_containers.yaml')
|
||||
env_file = os.path.join(temp, 'containers_env.yaml')
|
||||
tmpl_file = os.path.join(temp, 'overcloud_containers.yaml.j2')
|
||||
aodh_file = os.path.join(temp, 'docker', 'services',
|
||||
'overcloud_containers.yaml.j2')
|
||||
|
||||
resource_registry = {'resource_registry': {
|
||||
'OS::TripleO::Services::AodhEvaluator': aodh_file,
|
||||
'OS::TripleO::Services::AodhApi': aodh_file
|
||||
}}
|
||||
pmef.return_value = None, resource_registry
|
||||
|
||||
arglist = [
|
||||
'--template-file',
|
||||
|
@ -120,6 +130,8 @@ class TestContainerImagePrepare(TestPluginV1):
|
|||
images_file,
|
||||
'--env-file',
|
||||
env_file,
|
||||
'-e',
|
||||
'environment/docker.yaml'
|
||||
]
|
||||
self.cmd.app.command_options = arglist
|
||||
verifylist = []
|
||||
|
@ -127,9 +139,16 @@ class TestContainerImagePrepare(TestPluginV1):
|
|||
cift.return_value = [{
|
||||
'imagename': 'tripleo/os-aodh-apifoo:passed-ci',
|
||||
'params': ['DockerAodhApiImage', 'DockerAodhConfigImage'],
|
||||
'services': [
|
||||
'OS::TripleO::Services::AodhApi',
|
||||
'OS::TripleO::Services::AodhEvaluator',
|
||||
],
|
||||
}, {
|
||||
'imagename': 'tripleo/os-heat-apifoo:passed-ci',
|
||||
'params': ['DockerHeatApiImage'],
|
||||
'imagename': 'tripleo/os-aodh-evaluatorfoo:passed-ci',
|
||||
'params': ['DockerAodhEvaluatorImage'],
|
||||
'services': [
|
||||
'OS::TripleO::Services::AodhEvaluator',
|
||||
],
|
||||
}]
|
||||
|
||||
mock_builder.return_value.container_images_from_template = cift
|
||||
|
@ -139,6 +158,7 @@ class TestContainerImagePrepare(TestPluginV1):
|
|||
self.cmd.take_action(parsed_args)
|
||||
|
||||
mock_builder.assert_called_once_with([tmpl_file])
|
||||
pmef.assert_called_once_with(['environment/docker.yaml'])
|
||||
cift.assert_called_once_with(
|
||||
filter=mock.ANY,
|
||||
name_prefix='os-',
|
||||
|
@ -150,14 +170,15 @@ class TestContainerImagePrepare(TestPluginV1):
|
|||
'container_images': [{
|
||||
'imagename': 'tripleo/os-aodh-apifoo:passed-ci',
|
||||
}, {
|
||||
'imagename': 'tripleo/os-heat-apifoo:passed-ci',
|
||||
'imagename': 'tripleo/os-aodh-evaluatorfoo:passed-ci',
|
||||
}]
|
||||
}
|
||||
env_data = {
|
||||
'parameter_defaults': {
|
||||
'DockerAodhApiImage': 'tripleo/os-aodh-apifoo:passed-ci',
|
||||
'DockerAodhConfigImage': 'tripleo/os-aodh-apifoo:passed-ci',
|
||||
'DockerHeatApiImage': 'tripleo/os-heat-apifoo:passed-ci',
|
||||
'DockerAodhEvaluatorImage':
|
||||
'tripleo/os-aodh-evaluatorfoo:passed-ci'
|
||||
}
|
||||
}
|
||||
with open(images_file) as f:
|
||||
|
|
|
@ -21,6 +21,7 @@ import re
|
|||
import sys
|
||||
import tempfile
|
||||
|
||||
from heatclient.common import template_utils
|
||||
from osc_lib.command import command
|
||||
from osc_lib.i18n import _
|
||||
import yaml
|
||||
|
@ -235,6 +236,14 @@ class PrepareImageFiles(command.Command):
|
|||
help=_("File to write resulting image entries to, as well as "
|
||||
"stdout. Any existing file will be overwritten."),
|
||||
)
|
||||
parser.add_argument(
|
||||
'--service-environment-file', '-e', metavar='<file path>',
|
||||
action='append', dest='environment_files',
|
||||
help=_('Environment files specifying which services are '
|
||||
'containerized. Entries will be filtered to only contain '
|
||||
'images used by containerized services. (Can be specified '
|
||||
'more than once.)')
|
||||
)
|
||||
parser.add_argument(
|
||||
"--env-file",
|
||||
dest="env_file",
|
||||
|
@ -244,24 +253,34 @@ class PrepareImageFiles(command.Command):
|
|||
)
|
||||
return parser
|
||||
|
||||
def write_env_file(self, result, env_file):
|
||||
params = {}
|
||||
for entry in result:
|
||||
imagename = entry.get('imagename', '')
|
||||
if 'params' in entry:
|
||||
for p in entry.pop('params'):
|
||||
params[p] = imagename
|
||||
def write_env_file(self, params, env_file):
|
||||
|
||||
with os.fdopen(os.open(env_file,
|
||||
os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o666),
|
||||
'w') as f:
|
||||
f.write('# Generated with the following on %s\n#\n' %
|
||||
datetime.datetime.now().isoformat())
|
||||
f.write('# %s\n#\n\n' % ' '.join(self.app.command_options))
|
||||
f.write('# openstack %s\n#\n\n' %
|
||||
' '.join(self.app.command_options))
|
||||
|
||||
yaml.safe_dump({'parameter_defaults': params}, f,
|
||||
default_flow_style=False)
|
||||
|
||||
def build_service_filter(self, environment_files):
|
||||
if not environment_files:
|
||||
return None
|
||||
|
||||
service_filter = set()
|
||||
env_files, env = (
|
||||
template_utils.process_multiple_environments_and_files(
|
||||
environment_files))
|
||||
for service, env_path in env.get('resource_registry', {}).items():
|
||||
# Use the template path to determine if it represents a
|
||||
# containerized service
|
||||
if '/docker/services/' in env_path:
|
||||
service_filter.add(service)
|
||||
return service_filter
|
||||
|
||||
def take_action(self, parsed_args):
|
||||
self.log.debug("take_action(%s)" % parsed_args)
|
||||
subs = {
|
||||
|
@ -271,11 +290,19 @@ class PrepareImageFiles(command.Command):
|
|||
'name_suffix': parsed_args.suffix,
|
||||
}
|
||||
|
||||
service_filter = self.build_service_filter(
|
||||
parsed_args.environment_files)
|
||||
|
||||
def ffunc(entry):
|
||||
imagename = entry.get('imagename', '')
|
||||
for p in parsed_args.excludes:
|
||||
if re.search(p, imagename):
|
||||
return None
|
||||
if service_filter is not None:
|
||||
# check the entry is for a service being deployed
|
||||
image_services = set(entry.get('services', []))
|
||||
if not service_filter.intersection(image_services):
|
||||
return None
|
||||
if parsed_args.pull_source:
|
||||
entry['pull_source'] = parsed_args.pull_source
|
||||
if parsed_args.push_destination:
|
||||
|
@ -285,8 +312,17 @@ class PrepareImageFiles(command.Command):
|
|||
builder = kolla_builder.KollaImageBuilder([parsed_args.template_file])
|
||||
result = builder.container_images_from_template(filter=ffunc, **subs)
|
||||
|
||||
params = {}
|
||||
for entry in result:
|
||||
imagename = entry.get('imagename', '')
|
||||
if 'params' in entry:
|
||||
for p in entry.pop('params'):
|
||||
params[p] = imagename
|
||||
if 'services' in entry:
|
||||
del(entry['services'])
|
||||
|
||||
if parsed_args.env_file:
|
||||
self.write_env_file(result, parsed_args.env_file)
|
||||
self.write_env_file(params, parsed_args.env_file)
|
||||
|
||||
result_str = yaml.safe_dump({'container_images': result},
|
||||
default_flow_style=False)
|
||||
|
|
Loading…
Reference in New Issue