Merge "Support for Buildah in kolla_builder"

releasenotes/notes/buildah_build-727eb0f35f819731.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    kolla_builder now supports Buildah and not just Docker.

tripleo_common/constants.py

@@ -67,6 +67,9 @@ LOCAL_CACERT_PATH = '/etc/pki/ca-trust/source/anchors/cm-local-ca.pem'
 # Swift via SwiftPlanStorageBackend to identify them from other containers
 TRIPLEO_META_USAGE_KEY = 'x-container-meta-usage-tripleo'
 
+# 30 minutes maximum to build the child layers at the same time.
+BUILD_TIMEOUT = 1800
+
 #: List of names of parameters that contain passwords
 PASSWORD_PARAMETER_NAMES = (
     'AdminPassword',

tripleo_common/image/builder/base.py

@@ -0,0 +1,25 @@
+#   Copyright 2019 Red Hat, Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License"); you may
+#   not use this file except in compliance with the License. You may obtain
+#   a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#   License for the specific language governing permissions and limitations
+#   under the License.
+#
+
+
+class BaseBuilder(object):
+    """Base Tripleo-Common Image Builder.
+
+    For now it does nothing but this interface will allow
+    to support multiple builders and not just buildah or docker.
+    """
+
+    def __init__(self):
+        pass

tripleo_common/image/builder/buildah.py

@@ -0,0 +1,172 @@
+#   Copyright 2019 Red Hat, Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License"); you may
+#   not use this file except in compliance with the License. You may obtain
+#   a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#   License for the specific language governing permissions and limitations
+#   under the License.
+#
+
+
+from concurrent import futures
+import os
+import six
+
+from oslo_concurrency import processutils
+from oslo_log import log as logging
+
+from tripleo_common import constants
+from tripleo_common.image.builder import base
+from tripleo_common.utils import process
+
+LOG = logging.getLogger(__name__)
+
+
+class BuildahBuilder(base.BaseBuilder):
+    """Builder to build container images with Buildah."""
+
+    def __init__(self, work_dir, deps, base='fedora', img_type='binary',
+                 tag='latest', namespace='master',
+                 registry_address='127.0.0.1:8787'):
+        """Setup the parameters to build with Buildah.
+
+        :params work_dir: Directory where the Dockerfiles
+            are generated by Kolla.
+        :params deps: Dictionary defining the container images
+            dependencies.
+        :params base: Base image on which the containers are built.
+            Default to fedora.
+        :params img_type: Method used to build the image. All TripleO images
+            are built from binary method.
+        :params tag: Tag used to identify the images that we build.
+            Default to latest.
+        :params namespace: Namespace used to build the containers.
+            Default to master.
+        :params registry_address: IP + port of the registry where we push
+            the images. Default is 127.0.0.1:8787.
+        """
+
+        super(BuildahBuilder, self).__init__()
+        self.build_timeout = constants.BUILD_TIMEOUT
+        self.work_dir = work_dir
+        self.deps = deps
+        self.base = base
+        self.img_type = img_type
+        self.tag = tag
+        self.namespace = namespace
+        self.registry_address = registry_address
+        # Each container image has a Dockerfile. Buildah needs to know
+        # the base directory later.
+        self.cont_map = {os.path.basename(root): root for root, dirs,
+                         fnames in os.walk(self.work_dir)
+                         if 'Dockerfile' in fnames}
+        # Building images with root so overlayfs is used, and not fuse-overlay
+        # from userspace, which would be slower.
+        self.buildah_cmd = ['sudo', 'buildah']
+
+    def _find_container_dir(self, container_name):
+        """Return the path of the Dockerfile directory.
+
+        :params container_name: Name of the container.
+        """
+
+        if container_name not in self.cont_map:
+            LOG.error('Container not found in Kolla '
+                      'deps: %s' % container_name)
+        return self.cont_map.get(container_name, '')
+
+    def _generate_container(self, container_name):
+        """Generate a container image by building and pushing the image.
+
+        :params container_name: Name of the container.
+        """
+
+        self.build(container_name, self._find_container_dir(container_name))
+        destination = "{}/{}/{}-{}-{}:{}".format(
+            self.registry_address,
+            self.namespace,
+            self.base,
+            self.img_type,
+            container_name,
+            self.tag
+        )
+        self.push(destination)
+
+    def build(self, container_name, container_build_path):
+        """Build an image from a given directory.
+
+        :params container_name: Name of the container.
+        :params container_build_path: Directory where the Dockerfile and other
+            files are located to build the image.
+        """
+
+        destination = "{}/{}/{}-{}-{}:{}".format(
+            self.registry_address,
+            self.namespace,
+            self.base,
+            self.img_type,
+            container_name,
+            self.tag
+        )
+        # 'buildah bud' is the command we want because Kolla uses Dockefile to
+        # build images.
+        # TODO(emilien): Stop ignoring TLS. The deployer should either secure
+        # the registry or add it to insecure_registries.
+        logfile = container_build_path + '/' + container_name + '-build.log'
+        args = self.buildah_cmd + ['bud', '--tls-verify=False', '--logfile',
+                                   logfile, '-t', destination,
+                                   container_build_path]
+        print("Building %s image with: %s" % (container_name, ' '.join(args)))
+        process.execute(*args, run_as_root=False, use_standard_locale=True)
+
+    def push(self, destination):
+        """Push an image to a container registry.
+
+        :params destination: URL to used to push the container. It contains
+            the registry address, namespace, base, img_type, container name
+            and tag.
+        """
+        # TODO(emilien): Stop ignoring TLS. The deployer should either secure
+        # the registry or add it to insecure_registries.
+        # TODO(emilien) We need to figure out how we can push to something
+        # else than a Docker registry.
+        args = self.buildah_cmd + ['push', '--tls-verify=False', destination,
+                                   'docker://' + destination]
+        print("Pushing %s image with: %s" % (destination, ' '.join(args)))
+        process.execute(*args, run_as_root=False, use_standard_locale=True)
+
+    def build_all(self, deps=None):
+        """Function that browse containers dependencies and build them.
+
+        :params deps: Dictionary defining the container images
+            dependencies.
+        """
+
+        if deps is None:
+            deps = self.deps
+        if isinstance(deps, (list,)):
+            # Only a list of images can be multi-processed because they
+            # are the last layer to build. Otherwise we could have issues
+            # to build multiple times the same layer.
+            # Number of workers will be based on CPU count with a min 2,
+            # max 8. Concurrency in Buildah isn't that great so it's not
+            # useful to go above 8.
+            workers = min(8, max(2, processutils.get_worker_count()))
+            with futures.ThreadPoolExecutor(max_workers=workers) as executor:
+                future_to_build = {executor.submit(self.build_all,
+                                   container): container for container in
+                                   deps}
+                futures.wait(future_to_build, timeout=self.build_timeout,
+                             return_when=futures.ALL_COMPLETED)
+        elif isinstance(deps, (dict,)):
+            for container in deps:
+                self._generate_container(container)
+                self.build_all(deps.get(container))
+        elif isinstance(deps, six.string_types):
+            self._generate_container(deps)

tripleo_common/image/kolla_builder.py

@@ -432,7 +432,8 @@ class KollaImageBuilder(base.BaseImageManager):
                 result.append(entry)
         return result
 
-    def build_images(self, kolla_config_files=None, excludes=[]):
+    def build_images(self, kolla_config_files=None, excludes=[],
+                     template_only=False, kolla_tmp_dir=None):
 
         cmd = ['kolla-build']
         if kolla_config_files:
@@ -452,6 +453,15 @@ class KollaImageBuilder(base.BaseImageManager):
             if image and image not in excludes:
                 cmd.append(image)
 
+        if template_only:
+            # build the dep list cmd line
+            cmd_deps = list(cmd)
+            cmd_deps.append('--list-dependencies')
+            # build the template only cmd line
+            cmd.append('--template-only')
+            cmd.append('--work-dir')
+            cmd.append(kolla_tmp_dir)
+
         self.logger.info('Running %s' % ' '.join(cmd))
         env = os.environ.copy()
         process = subprocess.Popen(cmd, env=env, stdout=subprocess.PIPE,
@@ -459,4 +469,16 @@ class KollaImageBuilder(base.BaseImageManager):
         out, err = process.communicate()
         if process.returncode != 0:
             raise subprocess.CalledProcessError(process.returncode, cmd, err)
+
+        if template_only:
+            self.logger.info('Running %s' % ' '.join(cmd_deps))
+            env = os.environ.copy()
+            process = subprocess.Popen(cmd_deps, env=env,
+                                       stdout=subprocess.PIPE,
+                                       stderr=subprocess.PIPE,
+                                       universal_newlines=True)
+            out, err = process.communicate()
+            if process.returncode != 0:
+                raise subprocess.CalledProcessError(process.returncode,
+                                                    cmd_deps, err)
         return out

tripleo_common/tests/image/builder/test_buildah.py

@@ -0,0 +1,56 @@
+#   Copyright 2019 Red Hat, Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License"); you may
+#   not use this file except in compliance with the License. You may obtain
+#   a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#   License for the specific language governing permissions and limitations
+#   under the License.
+#
+"""Unit tests for image.builder.buildah"""
+
+import copy
+import mock
+
+from tripleo_common.image.builder.buildah import BuildahBuilder as bb
+from tripleo_common.tests import base
+from tripleo_common.utils import process
+
+
+BUILDAH_CMD_BASE = ['sudo', 'buildah']
+DEPS = {"base"}
+WORK_DIR = '/tmp/kolla'
+
+
+class TestBuildahBuilder(base.TestCase):
+
+    @mock.patch.object(process, 'execute', autospec=True)
+    def test_build(self, mock_process):
+        args = copy.copy(BUILDAH_CMD_BASE)
+        dest = '127.0.0.1:8787/master/fedora-binary-fedora-base:latest'
+        container_build_path = WORK_DIR + '/' + 'fedora-base'
+        logfile = '/tmp/kolla/fedora-base/fedora-base-build.log'
+        buildah_cmd_build = ['bud', '--tls-verify=False', '--logfile',
+                             logfile, '-t', dest, container_build_path]
+        args.extend(buildah_cmd_build)
+        bb(WORK_DIR, DEPS).build('fedora-base', container_build_path)
+        mock_process.assert_called_once_with(
+            *args, run_as_root=False, use_standard_locale=True
+        )
+
+    @mock.patch.object(process, 'execute', autospec=True)
+    def test_push(self, mock_process):
+        args = copy.copy(BUILDAH_CMD_BASE)
+        dest = '127.0.0.1:8787/master/fedora-binary-fedora-base:latest'
+        buildah_cmd_push = ['push', '--tls-verify=False', dest,
+                            'docker://' + dest]
+        args.extend(buildah_cmd_push)
+        bb(WORK_DIR, DEPS).push(dest)
+        mock_process.assert_called_once_with(
+            *args, run_as_root=False, use_standard_locale=True
+        )

tripleo_common/tests/image/test_kolla_builder.py

@@ -124,6 +124,45 @@ class TestKollaImageBuilder(base.TestCase):
             'image-with-missing-tag',
         ], env=env, stdout=-1, universal_newlines=True)
 
+    @mock.patch('tripleo_common.image.base.open',
+                mock.mock_open(read_data=filedata), create=True)
+    @mock.patch('os.path.isfile', return_value=True)
+    @mock.patch('subprocess.Popen')
+    def test_build_images_template_only(self, mock_popen, mock_path):
+        process = mock.Mock()
+        process.returncode = 0
+        process.communicate.return_value = 'done', ''
+        mock_popen.return_value = process
+
+        builder = kb.KollaImageBuilder(self.filelist)
+        self.assertEqual('done',
+                         builder.build_images(
+                             ['kolla-config.conf'], [], True, '/tmp/kolla'))
+        env = os.environ.copy()
+        call1 = mock.call([
+            'kolla-build',
+            '--config-file',
+            'kolla-config.conf',
+            'nova-compute',
+            'nova-libvirt',
+            'heat-docker-agents-centos',
+            'image-with-missing-tag',
+            '--template-only',
+            '--work-dir', '/tmp/kolla',
+        ], env=env, stdout=-1, universal_newlines=True)
+        call2 = mock.call([
+            'kolla-build',
+            '--config-file',
+            'kolla-config.conf',
+            'nova-compute',
+            'nova-libvirt',
+            'heat-docker-agents-centos',
+            'image-with-missing-tag',
+            '--list-dependencies',
+        ], env=env, stdout=-1, stderr=-1, universal_newlines=True)
+        calls = [call1, call2]
+        mock_popen.assert_has_calls(calls, any_order=True)
+
     @mock.patch('subprocess.Popen')
     def test_build_images_no_conf(self, mock_popen):
         process = mock.Mock()

tripleo_common/tests/utils/test_process.py

@@ -0,0 +1,81 @@
+# Copyright (c) 2019 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Unit tests for utils.process."""
+
+
+import os
+
+import mock
+from oslo_concurrency import processutils
+
+from tripleo_common.tests import base
+from tripleo_common.utils import process
+
+
+class ExecuteTestCase(base.TestCase):
+    # Allow calls to process.execute() and related functions
+    block_execute = False
+
+    @mock.patch.object(processutils, 'execute', autospec=True)
+    @mock.patch.object(os.environ, 'copy', return_value={}, autospec=True)
+    def test_execute_use_standard_locale_no_env_variables(self, env_mock,
+                                                          execute_mock):
+        process.execute('foo', use_standard_locale=True)
+        execute_mock.assert_called_once_with('foo',
+                                             env_variables={'LC_ALL': 'C'})
+
+    @mock.patch.object(processutils, 'execute', autospec=True)
+    def test_execute_use_standard_locale_with_env_variables(self,
+                                                            execute_mock):
+        process.execute('foo', use_standard_locale=True,
+                        env_variables={'foo': 'bar'})
+        execute_mock.assert_called_once_with('foo',
+                                             env_variables={'LC_ALL': 'C',
+                                                            'foo': 'bar'})
+
+    @mock.patch.object(processutils, 'execute', autospec=True)
+    def test_execute_not_use_standard_locale(self, execute_mock):
+        process.execute('foo', use_standard_locale=False,
+                        env_variables={'foo': 'bar'})
+        execute_mock.assert_called_once_with('foo',
+                                             env_variables={'foo': 'bar'})
+
+    @mock.patch.object(process, 'LOG', autospec=True)
+    def _test_execute_with_log_stdout(self, log_mock, log_stdout=None):
+        with mock.patch.object(
+                processutils, 'execute', autospec=True) as execute_mock:
+            execute_mock.return_value = ('stdout', 'stderr')
+            if log_stdout is not None:
+                process.execute('foo', log_stdout=log_stdout)
+            else:
+                process.execute('foo')
+            execute_mock.assert_called_once_with('foo')
+            name, args, kwargs = log_mock.debug.mock_calls[1]
+            if log_stdout is False:
+                self.assertEqual(2, log_mock.debug.call_count)
+                self.assertNotIn('stdout', args[0])
+            else:
+                self.assertEqual(3, log_mock.debug.call_count)
+                self.assertIn('stdout', args[0])
+
+    def test_execute_with_log_stdout_default(self):
+        self._test_execute_with_log_stdout()
+
+    def test_execute_with_log_stdout_true(self):
+        self._test_execute_with_log_stdout(log_stdout=True)
+
+    def test_execute_with_log_stdout_false(self):
+        self._test_execute_with_log_stdout(log_stdout=False)

tripleo_common/utils/process.py

@@ -0,0 +1,58 @@
+# Copyright (c) 2019 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Utilities to handle processes."""
+
+import logging
+import os
+
+from oslo_concurrency import processutils
+
+LOG = logging.getLogger(__name__)
+
+
+def execute(*cmd, **kwargs):
+    """Convenience wrapper around oslo's execute() method.
+
+    Executes and logs results from a system command. See docs for
+    oslo_concurrency.processutils.execute for usage.
+
+    :param \*cmd: positional arguments to pass to processutils.execute()
+    :param use_standard_locale: keyword-only argument. True | False.
+                                Defaults to False. If set to True,
+                                execute command with standard locale
+                                added to environment variables.
+    :param log_stdout: keyword-only argument. True | False. Defaults
+                       to True. If set to True, logs the output.
+    :param \*\*kwargs: keyword arguments to pass to processutils.execute()
+    :returns: (stdout, stderr) from process execution
+    :raises: UnknownArgumentError on receiving unknown arguments
+    :raises: ProcessExecutionError
+    :raises: OSError
+    """
+
+    use_standard_locale = kwargs.pop('use_standard_locale', False)
+    if use_standard_locale:
+        env = kwargs.pop('env_variables', os.environ.copy())
+        env['LC_ALL'] = 'C'
+        kwargs['env_variables'] = env
+    log_stdout = kwargs.pop('log_stdout', True)
+    result = processutils.execute(*cmd, **kwargs)
+    LOG.debug('Execution completed, command line is "%s"',
+              ' '.join(map(str, cmd)))
+    if log_stdout:
+        LOG.debug('Command stdout is: "%s"', result[0])
+    LOG.debug('Command stderr is: "%s"', result[1])
+    return result