Support squash docker images layers

Implements: blueprint squash-layers
Change-Id: Ic9a144e50440ccb37f7781d8955815a64282e687

kolla/common/config.py

@@ -258,6 +258,10 @@ _CLI_OPTS = [
                 help='Attempt to pull a newer version of the base image'),
     cfg.StrOpt('work-dir', help=('Path to be used as working directory.'
                                  ' By default, a temporary dir is created')),
+    cfg.BoolOpt('squash', default=False,
+                help=('Squash the image layers. WARNING: it will consume lots'
+                      ' of disk IO. "docker-squash" tool is required, install'
+                      ' it by "pip install docker-squash"')),
 ]
 
 _BASE_OPTS = [
@@ -269,7 +273,11 @@ _BASE_OPTS = [
                 help=('Comma separated list of .rpm or .repo file(s) '
                       'or URL(s) to install before building containers')),
     cfg.StrOpt('apt_sources_list', help=('Path to custom sources.list')),
-    cfg.StrOpt('apt_preferences', help=('Path to custom apt/preferences'))
+    cfg.StrOpt('apt_preferences', help=('Path to custom apt/preferences')),
+    cfg.BoolOpt('squash-cleanup', default=True,
+                help='Remove source image from Docker after squashing'),
+    cfg.StrOpt('squash-tmp-dir',
+               help='Temporary directory to be used during squashing')
 ]
 
 

kolla/common/utils.py

@@ -0,0 +1,73 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+import subprocess  # nosec
+import sys
+
+
+def make_a_logger(conf=None, image_name=None):
+    if image_name:
+        log = logging.getLogger(".".join([__name__, image_name]))
+    else:
+        log = logging.getLogger(__name__)
+    if not log.handlers:
+        if conf is None or not conf.logs_dir or not image_name:
+            handler = logging.StreamHandler(sys.stderr)
+            log.propagate = False
+        else:
+            filename = os.path.join(conf.logs_dir, "%s.log" % image_name)
+            handler = logging.FileHandler(filename, delay=True)
+        handler.setFormatter(logging.Formatter(logging.BASIC_FORMAT))
+        log.addHandler(handler)
+    if conf is not None and conf.debug:
+        log.setLevel(logging.DEBUG)
+    else:
+        log.setLevel(logging.INFO)
+    return log
+
+
+LOG = make_a_logger()
+
+
+def get_docker_squash_version():
+
+    try:
+        stdout = subprocess.check_output(  # nosec
+            ['docker-squash', '--version'], stderr=subprocess.STDOUT)
+        return stdout.split()[0]
+    except OSError as ex:
+        if ex.errno == 2:
+            LOG.error(('"docker-squash" command is not found.'
+                       ' try to install it by "pip install docker-squash"'))
+        raise
+
+
+def squash(old_image, new_image,
+           from_layer=None,
+           cleanup=False,
+           tmp_dir=None):
+
+    cmds = ['docker-squash', '--tag', new_image, old_image]
+    if cleanup:
+        cmds += ['--cleanup']
+    if from_layer:
+        cmds += ['--from-layer', from_layer]
+    if tmp_dir:
+        cmds += ['--tmp-dir', tmp_dir]
+    try:
+        subprocess.check_output(cmds, stderr=subprocess.STDOUT)  # nosec
+    except subprocess.CalledProcessError as ex:
+        LOG.exception('Get error during squashing image: %s',
+                      ex.stdout)
+        raise

kolla/image/build.py

@@ -36,6 +36,7 @@ from oslo_config import cfg
 from requests import exceptions as requests_exc
 import six
 
+
 # NOTE(SamYaple): Update the search path to prefer PROJECT_ROOT as the source
 #                 of packages to import if we are using local tools instead of
 #                 pip installed kolla tools
@@ -46,34 +47,14 @@ if PROJECT_ROOT not in sys.path:
 
 from kolla.common import config as common_config
 from kolla.common import task
+from kolla.common import utils
 from kolla import exception
 from kolla.template import filters as jinja_filters
 from kolla.template import methods as jinja_methods
 from kolla import version
 
 
-def make_a_logger(conf=None, image_name=None):
-    if image_name:
-        log = logging.getLogger(".".join([__name__, image_name]))
-    else:
-        log = logging.getLogger(__name__)
-    if not log.handlers:
-        if conf is None or not conf.logs_dir or not image_name:
-            handler = logging.StreamHandler(sys.stderr)
-            log.propagate = False
-        else:
-            filename = os.path.join(conf.logs_dir, "%s.log" % image_name)
-            handler = logging.FileHandler(filename, delay=True)
-        handler.setFormatter(logging.Formatter(logging.BASIC_FORMAT))
-        log.addHandler(handler)
-    if conf is not None and conf.debug:
-        log.setLevel(logging.DEBUG)
-    else:
-        log.setLevel(logging.INFO)
-    return log
-
-
-LOG = make_a_logger()
+LOG = utils.make_a_logger()
 
 # Image status constants.
 #
@@ -261,7 +242,7 @@ class Image(object):
         self.source = source
         self.parent_name = parent_name
         if logger is None:
-            logger = make_a_logger(image_name=name)
+            logger = utils.make_a_logger(image_name=name)
         self.logger = logger
         self.children = []
         self.plugins = []
@@ -586,6 +567,9 @@ class BuildTask(DockerTask):
                         if line:
                             self.logger.error('%s', line)
                     return
+
+            if image.status != STATUS_ERROR and self.conf.squash:
+                self.squash()
         except docker.errors.DockerException:
             image.status = STATUS_ERROR
             self.logger.exception('Unknown docker error when building')
@@ -596,6 +580,19 @@ class BuildTask(DockerTask):
             image.status = STATUS_BUILT
             self.logger.info('Built')
 
+    def squash(self):
+        image_tag = self.image.canonical_name
+        image_id = self.dc.inspect_image(image_tag)['Id']
+
+        parent_history = self.dc.history(self.image.parent_name)
+        parent_last_layer = parent_history[0]['Id']
+        self.logger.info('Parent lastest layer is: %s' % parent_last_layer)
+
+        utils.squash(image_id, image_tag, from_layer=parent_last_layer,
+                     cleanup=self.conf.squash_cleanup,
+                     tmp_dir=self.conf.squash_tmp_dir)
+        self.logger.info('Image is squashed successfully')
+
 
 class WorkerThread(threading.Thread):
     """Thread that executes tasks until the queue provides a tombstone."""
@@ -1084,7 +1081,7 @@ class KollaWorker(object):
             del match
             image = Image(image_name, canonical_name, path,
                           parent_name=parent_name,
-                          logger=make_a_logger(self.conf, image_name),
+                          logger=utils.make_a_logger(self.conf, image_name),
                           docker_client=self.dc)
 
             if self.install_type == 'source':
@@ -1226,6 +1223,13 @@ def run_build():
     if conf.debug:
         LOG.setLevel(logging.DEBUG)
 
+    if conf.squash:
+        squash_version = utils.get_docker_squash_version()
+        LOG.info('Image squash is enabled and "docker-squash" version is %s',
+                 squash_version)
+    else:
+        LOG.info('Image squash is disabled')
+
     kolla = KollaWorker(conf)
     kolla.setup_working_dir()
     kolla.find_dockerfiles()

releasenotes/notes/support-docker-image-squash-8396c0de63085f5d.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    support --squash parameter which leverage docker-squash tool to squash
+    newly built layers into a single new layer