From b047a201352430c2e55ec38e10bd9859e004eddb Mon Sep 17 00:00:00 2001 From: Emilien Macchi Date: Thu, 31 Jan 2019 16:17:07 -0500 Subject: [PATCH] Support for Buildah in kolla_builder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Buildah and Podman will replace Docker. This patch will be used by tripleoclient when running: $ openstack overcloud container image build --use-buildah When using Buildah, the kolla_builder will, in that order: 1) Generate container templates but not actually build the images. The directories are generated by kolla-build and containers files like Dockerfiles and such. 2) Generate container dependencies and build a dictionary, later used by the new BuildahBuilder. In this patch, we introduce a Class for builders. For now, we only have BuildahBuilder but later we will refactor kolla_builder. The BuildahBuilder has in charge of: 1) Build containers using "buildah bud". This command is used because Kolla uses Dockerfiles to build images. Each image build is logged in the directory that contains the Dockerfile. During the build, logging displays the container that is being built and also the buildah command that is used. The image layers that don't have childs are multi-threaded to accelerate the build. We don't go over 8 builds at the same time otherwise Buildah struggles with the locks too hard. We also setup a timeout of 30 minutes for the workers to report back. For example: base └─openstack-base ├─nova-base │ ├─nova-api │ └─nova-conductor └─neutron-base └─neutron-dhcp └─multipathd └─crond The builder will first build "base" then: - build openstack-base, multipathd and crond in same time. - build nova-base and neutron-base in same time - build nova-api, nova-conductor in same time - etc 2) Push containers to a Docker registry. We'll support more than Docker registries, but later. Note: All commands are executed using processutils from oslo_concurrency which is pretty and rock solid. Note2: kolla_builder will be refactored to use the new Builder class. This patch is an initial support for Buildah, improvements will come later. Co-Authored-By: Alex Schultz Co-Authored-By: Christophe Fontaine blueprint podman-support Change-Id: Ieff41a5f84456530b4621218b01f3b546cd867bf --- .../notes/buildah_build-727eb0f35f819731.yaml | 4 + tripleo_common/constants.py | 3 + tripleo_common/image/builder/__init__.py | 0 tripleo_common/image/builder/base.py | 25 +++ tripleo_common/image/builder/buildah.py | 172 ++++++++++++++++++ tripleo_common/image/kolla_builder.py | 24 ++- .../tests/image/builder/__init__.py | 0 .../tests/image/builder/test_buildah.py | 56 ++++++ .../tests/image/test_kolla_builder.py | 39 ++++ tripleo_common/tests/utils/test_process.py | 81 +++++++++ tripleo_common/utils/process.py | 58 ++++++ 11 files changed, 461 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/buildah_build-727eb0f35f819731.yaml create mode 100644 tripleo_common/image/builder/__init__.py create mode 100644 tripleo_common/image/builder/base.py create mode 100644 tripleo_common/image/builder/buildah.py create mode 100644 tripleo_common/tests/image/builder/__init__.py create mode 100644 tripleo_common/tests/image/builder/test_buildah.py create mode 100644 tripleo_common/tests/utils/test_process.py create mode 100644 tripleo_common/utils/process.py diff --git a/releasenotes/notes/buildah_build-727eb0f35f819731.yaml b/releasenotes/notes/buildah_build-727eb0f35f819731.yaml new file mode 100644 index 000000000..125607f8c --- /dev/null +++ b/releasenotes/notes/buildah_build-727eb0f35f819731.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + kolla_builder now supports Buildah and not just Docker. diff --git a/tripleo_common/constants.py b/tripleo_common/constants.py index edefdfaf7..5349c70d3 100644 --- a/tripleo_common/constants.py +++ b/tripleo_common/constants.py @@ -67,6 +67,9 @@ LOCAL_CACERT_PATH = '/etc/pki/ca-trust/source/anchors/cm-local-ca.pem' # Swift via SwiftPlanStorageBackend to identify them from other containers TRIPLEO_META_USAGE_KEY = 'x-container-meta-usage-tripleo' +# 30 minutes maximum to build the child layers at the same time. +BUILD_TIMEOUT = 1800 + #: List of names of parameters that contain passwords PASSWORD_PARAMETER_NAMES = ( 'AdminPassword', diff --git a/tripleo_common/image/builder/__init__.py b/tripleo_common/image/builder/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tripleo_common/image/builder/base.py b/tripleo_common/image/builder/base.py new file mode 100644 index 000000000..9db19bd75 --- /dev/null +++ b/tripleo_common/image/builder/base.py @@ -0,0 +1,25 @@ +# Copyright 2019 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + + +class BaseBuilder(object): + """Base Tripleo-Common Image Builder. + + For now it does nothing but this interface will allow + to support multiple builders and not just buildah or docker. + """ + + def __init__(self): + pass diff --git a/tripleo_common/image/builder/buildah.py b/tripleo_common/image/builder/buildah.py new file mode 100644 index 000000000..1478f580f --- /dev/null +++ b/tripleo_common/image/builder/buildah.py @@ -0,0 +1,172 @@ +# Copyright 2019 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + + +from concurrent import futures +import os +import six + +from oslo_concurrency import processutils +from oslo_log import log as logging + +from tripleo_common import constants +from tripleo_common.image.builder import base +from tripleo_common.utils import process + +LOG = logging.getLogger(__name__) + + +class BuildahBuilder(base.BaseBuilder): + """Builder to build container images with Buildah.""" + + def __init__(self, work_dir, deps, base='fedora', img_type='binary', + tag='latest', namespace='master', + registry_address='127.0.0.1:8787'): + """Setup the parameters to build with Buildah. + + :params work_dir: Directory where the Dockerfiles + are generated by Kolla. + :params deps: Dictionary defining the container images + dependencies. + :params base: Base image on which the containers are built. + Default to fedora. + :params img_type: Method used to build the image. All TripleO images + are built from binary method. + :params tag: Tag used to identify the images that we build. + Default to latest. + :params namespace: Namespace used to build the containers. + Default to master. + :params registry_address: IP + port of the registry where we push + the images. Default is 127.0.0.1:8787. + """ + + super(BuildahBuilder, self).__init__() + self.build_timeout = constants.BUILD_TIMEOUT + self.work_dir = work_dir + self.deps = deps + self.base = base + self.img_type = img_type + self.tag = tag + self.namespace = namespace + self.registry_address = registry_address + # Each container image has a Dockerfile. Buildah needs to know + # the base directory later. + self.cont_map = {os.path.basename(root): root for root, dirs, + fnames in os.walk(self.work_dir) + if 'Dockerfile' in fnames} + # Building images with root so overlayfs is used, and not fuse-overlay + # from userspace, which would be slower. + self.buildah_cmd = ['sudo', 'buildah'] + + def _find_container_dir(self, container_name): + """Return the path of the Dockerfile directory. + + :params container_name: Name of the container. + """ + + if container_name not in self.cont_map: + LOG.error('Container not found in Kolla ' + 'deps: %s' % container_name) + return self.cont_map.get(container_name, '') + + def _generate_container(self, container_name): + """Generate a container image by building and pushing the image. + + :params container_name: Name of the container. + """ + + self.build(container_name, self._find_container_dir(container_name)) + destination = "{}/{}/{}-{}-{}:{}".format( + self.registry_address, + self.namespace, + self.base, + self.img_type, + container_name, + self.tag + ) + self.push(destination) + + def build(self, container_name, container_build_path): + """Build an image from a given directory. + + :params container_name: Name of the container. + :params container_build_path: Directory where the Dockerfile and other + files are located to build the image. + """ + + destination = "{}/{}/{}-{}-{}:{}".format( + self.registry_address, + self.namespace, + self.base, + self.img_type, + container_name, + self.tag + ) + # 'buildah bud' is the command we want because Kolla uses Dockefile to + # build images. + # TODO(emilien): Stop ignoring TLS. The deployer should either secure + # the registry or add it to insecure_registries. + logfile = container_build_path + '/' + container_name + '-build.log' + args = self.buildah_cmd + ['bud', '--tls-verify=False', '--logfile', + logfile, '-t', destination, + container_build_path] + print("Building %s image with: %s" % (container_name, ' '.join(args))) + process.execute(*args, run_as_root=False, use_standard_locale=True) + + def push(self, destination): + """Push an image to a container registry. + + :params destination: URL to used to push the container. It contains + the registry address, namespace, base, img_type, container name + and tag. + """ + # TODO(emilien): Stop ignoring TLS. The deployer should either secure + # the registry or add it to insecure_registries. + # TODO(emilien) We need to figure out how we can push to something + # else than a Docker registry. + args = self.buildah_cmd + ['push', '--tls-verify=False', destination, + 'docker://' + destination] + print("Pushing %s image with: %s" % (destination, ' '.join(args))) + process.execute(*args, run_as_root=False, use_standard_locale=True) + + def build_all(self, deps=None): + """Function that browse containers dependencies and build them. + + :params deps: Dictionary defining the container images + dependencies. + """ + + if deps is None: + deps = self.deps + if isinstance(deps, (list,)): + # Only a list of images can be multi-processed because they + # are the last layer to build. Otherwise we could have issues + # to build multiple times the same layer. + # Number of workers will be based on CPU count with a min 2, + # max 8. Concurrency in Buildah isn't that great so it's not + # useful to go above 8. + workers = min(8, max(2, processutils.get_worker_count())) + with futures.ThreadPoolExecutor(max_workers=workers) as executor: + future_to_build = {executor.submit(self.build_all, + container): container for container in + deps} + futures.wait(future_to_build, timeout=self.build_timeout, + return_when=futures.ALL_COMPLETED) + elif isinstance(deps, (dict,)): + for container in deps: + self._generate_container(container) + self.build_all(deps.get(container)) + elif isinstance(deps, six.string_types): + self._generate_container(deps) diff --git a/tripleo_common/image/kolla_builder.py b/tripleo_common/image/kolla_builder.py index 893f6f267..3382f29dd 100644 --- a/tripleo_common/image/kolla_builder.py +++ b/tripleo_common/image/kolla_builder.py @@ -432,7 +432,8 @@ class KollaImageBuilder(base.BaseImageManager): result.append(entry) return result - def build_images(self, kolla_config_files=None, excludes=[]): + def build_images(self, kolla_config_files=None, excludes=[], + template_only=False, kolla_tmp_dir=None): cmd = ['kolla-build'] if kolla_config_files: @@ -452,6 +453,15 @@ class KollaImageBuilder(base.BaseImageManager): if image and image not in excludes: cmd.append(image) + if template_only: + # build the dep list cmd line + cmd_deps = list(cmd) + cmd_deps.append('--list-dependencies') + # build the template only cmd line + cmd.append('--template-only') + cmd.append('--work-dir') + cmd.append(kolla_tmp_dir) + self.logger.info('Running %s' % ' '.join(cmd)) env = os.environ.copy() process = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE, @@ -459,4 +469,16 @@ class KollaImageBuilder(base.BaseImageManager): out, err = process.communicate() if process.returncode != 0: raise subprocess.CalledProcessError(process.returncode, cmd, err) + + if template_only: + self.logger.info('Running %s' % ' '.join(cmd_deps)) + env = os.environ.copy() + process = subprocess.Popen(cmd_deps, env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True) + out, err = process.communicate() + if process.returncode != 0: + raise subprocess.CalledProcessError(process.returncode, + cmd_deps, err) return out diff --git a/tripleo_common/tests/image/builder/__init__.py b/tripleo_common/tests/image/builder/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tripleo_common/tests/image/builder/test_buildah.py b/tripleo_common/tests/image/builder/test_buildah.py new file mode 100644 index 000000000..2ad787aa8 --- /dev/null +++ b/tripleo_common/tests/image/builder/test_buildah.py @@ -0,0 +1,56 @@ +# Copyright 2019 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +"""Unit tests for image.builder.buildah""" + +import copy +import mock + +from tripleo_common.image.builder.buildah import BuildahBuilder as bb +from tripleo_common.tests import base +from tripleo_common.utils import process + + +BUILDAH_CMD_BASE = ['sudo', 'buildah'] +DEPS = {"base"} +WORK_DIR = '/tmp/kolla' + + +class TestBuildahBuilder(base.TestCase): + + @mock.patch.object(process, 'execute', autospec=True) + def test_build(self, mock_process): + args = copy.copy(BUILDAH_CMD_BASE) + dest = '127.0.0.1:8787/master/fedora-binary-fedora-base:latest' + container_build_path = WORK_DIR + '/' + 'fedora-base' + logfile = '/tmp/kolla/fedora-base/fedora-base-build.log' + buildah_cmd_build = ['bud', '--tls-verify=False', '--logfile', + logfile, '-t', dest, container_build_path] + args.extend(buildah_cmd_build) + bb(WORK_DIR, DEPS).build('fedora-base', container_build_path) + mock_process.assert_called_once_with( + *args, run_as_root=False, use_standard_locale=True + ) + + @mock.patch.object(process, 'execute', autospec=True) + def test_push(self, mock_process): + args = copy.copy(BUILDAH_CMD_BASE) + dest = '127.0.0.1:8787/master/fedora-binary-fedora-base:latest' + buildah_cmd_push = ['push', '--tls-verify=False', dest, + 'docker://' + dest] + args.extend(buildah_cmd_push) + bb(WORK_DIR, DEPS).push(dest) + mock_process.assert_called_once_with( + *args, run_as_root=False, use_standard_locale=True + ) diff --git a/tripleo_common/tests/image/test_kolla_builder.py b/tripleo_common/tests/image/test_kolla_builder.py index ee648e96c..2cae8e5a4 100644 --- a/tripleo_common/tests/image/test_kolla_builder.py +++ b/tripleo_common/tests/image/test_kolla_builder.py @@ -124,6 +124,45 @@ class TestKollaImageBuilder(base.TestCase): 'image-with-missing-tag', ], env=env, stdout=-1, universal_newlines=True) + @mock.patch('tripleo_common.image.base.open', + mock.mock_open(read_data=filedata), create=True) + @mock.patch('os.path.isfile', return_value=True) + @mock.patch('subprocess.Popen') + def test_build_images_template_only(self, mock_popen, mock_path): + process = mock.Mock() + process.returncode = 0 + process.communicate.return_value = 'done', '' + mock_popen.return_value = process + + builder = kb.KollaImageBuilder(self.filelist) + self.assertEqual('done', + builder.build_images( + ['kolla-config.conf'], [], True, '/tmp/kolla')) + env = os.environ.copy() + call1 = mock.call([ + 'kolla-build', + '--config-file', + 'kolla-config.conf', + 'nova-compute', + 'nova-libvirt', + 'heat-docker-agents-centos', + 'image-with-missing-tag', + '--template-only', + '--work-dir', '/tmp/kolla', + ], env=env, stdout=-1, universal_newlines=True) + call2 = mock.call([ + 'kolla-build', + '--config-file', + 'kolla-config.conf', + 'nova-compute', + 'nova-libvirt', + 'heat-docker-agents-centos', + 'image-with-missing-tag', + '--list-dependencies', + ], env=env, stdout=-1, stderr=-1, universal_newlines=True) + calls = [call1, call2] + mock_popen.assert_has_calls(calls, any_order=True) + @mock.patch('subprocess.Popen') def test_build_images_no_conf(self, mock_popen): process = mock.Mock() diff --git a/tripleo_common/tests/utils/test_process.py b/tripleo_common/tests/utils/test_process.py new file mode 100644 index 000000000..e9a66d4da --- /dev/null +++ b/tripleo_common/tests/utils/test_process.py @@ -0,0 +1,81 @@ +# Copyright (c) 2019 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for utils.process.""" + + +import os + +import mock +from oslo_concurrency import processutils + +from tripleo_common.tests import base +from tripleo_common.utils import process + + +class ExecuteTestCase(base.TestCase): + # Allow calls to process.execute() and related functions + block_execute = False + + @mock.patch.object(processutils, 'execute', autospec=True) + @mock.patch.object(os.environ, 'copy', return_value={}, autospec=True) + def test_execute_use_standard_locale_no_env_variables(self, env_mock, + execute_mock): + process.execute('foo', use_standard_locale=True) + execute_mock.assert_called_once_with('foo', + env_variables={'LC_ALL': 'C'}) + + @mock.patch.object(processutils, 'execute', autospec=True) + def test_execute_use_standard_locale_with_env_variables(self, + execute_mock): + process.execute('foo', use_standard_locale=True, + env_variables={'foo': 'bar'}) + execute_mock.assert_called_once_with('foo', + env_variables={'LC_ALL': 'C', + 'foo': 'bar'}) + + @mock.patch.object(processutils, 'execute', autospec=True) + def test_execute_not_use_standard_locale(self, execute_mock): + process.execute('foo', use_standard_locale=False, + env_variables={'foo': 'bar'}) + execute_mock.assert_called_once_with('foo', + env_variables={'foo': 'bar'}) + + @mock.patch.object(process, 'LOG', autospec=True) + def _test_execute_with_log_stdout(self, log_mock, log_stdout=None): + with mock.patch.object( + processutils, 'execute', autospec=True) as execute_mock: + execute_mock.return_value = ('stdout', 'stderr') + if log_stdout is not None: + process.execute('foo', log_stdout=log_stdout) + else: + process.execute('foo') + execute_mock.assert_called_once_with('foo') + name, args, kwargs = log_mock.debug.mock_calls[1] + if log_stdout is False: + self.assertEqual(2, log_mock.debug.call_count) + self.assertNotIn('stdout', args[0]) + else: + self.assertEqual(3, log_mock.debug.call_count) + self.assertIn('stdout', args[0]) + + def test_execute_with_log_stdout_default(self): + self._test_execute_with_log_stdout() + + def test_execute_with_log_stdout_true(self): + self._test_execute_with_log_stdout(log_stdout=True) + + def test_execute_with_log_stdout_false(self): + self._test_execute_with_log_stdout(log_stdout=False) diff --git a/tripleo_common/utils/process.py b/tripleo_common/utils/process.py new file mode 100644 index 000000000..16fdc01da --- /dev/null +++ b/tripleo_common/utils/process.py @@ -0,0 +1,58 @@ +# Copyright (c) 2019 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Utilities to handle processes.""" + +import logging +import os + +from oslo_concurrency import processutils + +LOG = logging.getLogger(__name__) + + +def execute(*cmd, **kwargs): + """Convenience wrapper around oslo's execute() method. + + Executes and logs results from a system command. See docs for + oslo_concurrency.processutils.execute for usage. + + :param \*cmd: positional arguments to pass to processutils.execute() + :param use_standard_locale: keyword-only argument. True | False. + Defaults to False. If set to True, + execute command with standard locale + added to environment variables. + :param log_stdout: keyword-only argument. True | False. Defaults + to True. If set to True, logs the output. + :param \*\*kwargs: keyword arguments to pass to processutils.execute() + :returns: (stdout, stderr) from process execution + :raises: UnknownArgumentError on receiving unknown arguments + :raises: ProcessExecutionError + :raises: OSError + """ + + use_standard_locale = kwargs.pop('use_standard_locale', False) + if use_standard_locale: + env = kwargs.pop('env_variables', os.environ.copy()) + env['LC_ALL'] = 'C' + kwargs['env_variables'] = env + log_stdout = kwargs.pop('log_stdout', True) + result = processutils.execute(*cmd, **kwargs) + LOG.debug('Execution completed, command line is "%s"', + ' '.join(map(str, cmd))) + if log_stdout: + LOG.debug('Command stdout is: "%s"', result[0]) + LOG.debug('Command stderr is: "%s"', result[1]) + return result