From d98a102162ff76b09771585ec21ca23415335b90 Mon Sep 17 00:00:00 2001 From: Jeffrey Zhang Date: Wed, 6 Dec 2017 12:20:26 +0800 Subject: [PATCH] Support squash docker images layers Implements: blueprint squash-layers Change-Id: Ic9a144e50440ccb37f7781d8955815a64282e687 --- kolla/common/config.py | 10 ++- kolla/common/utils.py | 73 +++++++++++++++++++ kolla/image/build.py | 52 +++++++------ ...-docker-image-squash-8396c0de63085f5d.yaml | 5 ++ 4 files changed, 115 insertions(+), 25 deletions(-) create mode 100644 kolla/common/utils.py create mode 100644 releasenotes/notes/support-docker-image-squash-8396c0de63085f5d.yaml diff --git a/kolla/common/config.py b/kolla/common/config.py index f7e3d1be35..f2e57eef02 100755 --- a/kolla/common/config.py +++ b/kolla/common/config.py @@ -258,6 +258,10 @@ _CLI_OPTS = [ help='Attempt to pull a newer version of the base image'), cfg.StrOpt('work-dir', help=('Path to be used as working directory.' ' By default, a temporary dir is created')), + cfg.BoolOpt('squash', default=False, + help=('Squash the image layers. WARNING: it will consume lots' + ' of disk IO. "docker-squash" tool is required, install' + ' it by "pip install docker-squash"')), ] _BASE_OPTS = [ @@ -269,7 +273,11 @@ _BASE_OPTS = [ help=('Comma separated list of .rpm or .repo file(s) ' 'or URL(s) to install before building containers')), cfg.StrOpt('apt_sources_list', help=('Path to custom sources.list')), - cfg.StrOpt('apt_preferences', help=('Path to custom apt/preferences')) + cfg.StrOpt('apt_preferences', help=('Path to custom apt/preferences')), + cfg.BoolOpt('squash-cleanup', default=True, + help='Remove source image from Docker after squashing'), + cfg.StrOpt('squash-tmp-dir', + help='Temporary directory to be used during squashing') ] diff --git a/kolla/common/utils.py b/kolla/common/utils.py new file mode 100644 index 0000000000..bb499dfe50 --- /dev/null +++ b/kolla/common/utils.py @@ -0,0 +1,73 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import subprocess # nosec +import sys + + +def make_a_logger(conf=None, image_name=None): + if image_name: + log = logging.getLogger(".".join([__name__, image_name])) + else: + log = logging.getLogger(__name__) + if not log.handlers: + if conf is None or not conf.logs_dir or not image_name: + handler = logging.StreamHandler(sys.stderr) + log.propagate = False + else: + filename = os.path.join(conf.logs_dir, "%s.log" % image_name) + handler = logging.FileHandler(filename, delay=True) + handler.setFormatter(logging.Formatter(logging.BASIC_FORMAT)) + log.addHandler(handler) + if conf is not None and conf.debug: + log.setLevel(logging.DEBUG) + else: + log.setLevel(logging.INFO) + return log + + +LOG = make_a_logger() + + +def get_docker_squash_version(): + + try: + stdout = subprocess.check_output( # nosec + ['docker-squash', '--version'], stderr=subprocess.STDOUT) + return stdout.split()[0] + except OSError as ex: + if ex.errno == 2: + LOG.error(('"docker-squash" command is not found.' + ' try to install it by "pip install docker-squash"')) + raise + + +def squash(old_image, new_image, + from_layer=None, + cleanup=False, + tmp_dir=None): + + cmds = ['docker-squash', '--tag', new_image, old_image] + if cleanup: + cmds += ['--cleanup'] + if from_layer: + cmds += ['--from-layer', from_layer] + if tmp_dir: + cmds += ['--tmp-dir', tmp_dir] + try: + subprocess.check_output(cmds, stderr=subprocess.STDOUT) # nosec + except subprocess.CalledProcessError as ex: + LOG.exception('Get error during squashing image: %s', + ex.stdout) + raise diff --git a/kolla/image/build.py b/kolla/image/build.py index a6538ff956..b70b201bd2 100755 --- a/kolla/image/build.py +++ b/kolla/image/build.py @@ -36,6 +36,7 @@ from oslo_config import cfg from requests import exceptions as requests_exc import six + # NOTE(SamYaple): Update the search path to prefer PROJECT_ROOT as the source # of packages to import if we are using local tools instead of # pip installed kolla tools @@ -46,34 +47,14 @@ if PROJECT_ROOT not in sys.path: from kolla.common import config as common_config from kolla.common import task +from kolla.common import utils from kolla import exception from kolla.template import filters as jinja_filters from kolla.template import methods as jinja_methods from kolla import version -def make_a_logger(conf=None, image_name=None): - if image_name: - log = logging.getLogger(".".join([__name__, image_name])) - else: - log = logging.getLogger(__name__) - if not log.handlers: - if conf is None or not conf.logs_dir or not image_name: - handler = logging.StreamHandler(sys.stderr) - log.propagate = False - else: - filename = os.path.join(conf.logs_dir, "%s.log" % image_name) - handler = logging.FileHandler(filename, delay=True) - handler.setFormatter(logging.Formatter(logging.BASIC_FORMAT)) - log.addHandler(handler) - if conf is not None and conf.debug: - log.setLevel(logging.DEBUG) - else: - log.setLevel(logging.INFO) - return log - - -LOG = make_a_logger() +LOG = utils.make_a_logger() # Image status constants. # @@ -261,7 +242,7 @@ class Image(object): self.source = source self.parent_name = parent_name if logger is None: - logger = make_a_logger(image_name=name) + logger = utils.make_a_logger(image_name=name) self.logger = logger self.children = [] self.plugins = [] @@ -586,6 +567,9 @@ class BuildTask(DockerTask): if line: self.logger.error('%s', line) return + + if image.status != STATUS_ERROR and self.conf.squash: + self.squash() except docker.errors.DockerException: image.status = STATUS_ERROR self.logger.exception('Unknown docker error when building') @@ -596,6 +580,19 @@ class BuildTask(DockerTask): image.status = STATUS_BUILT self.logger.info('Built') + def squash(self): + image_tag = self.image.canonical_name + image_id = self.dc.inspect_image(image_tag)['Id'] + + parent_history = self.dc.history(self.image.parent_name) + parent_last_layer = parent_history[0]['Id'] + self.logger.info('Parent lastest layer is: %s' % parent_last_layer) + + utils.squash(image_id, image_tag, from_layer=parent_last_layer, + cleanup=self.conf.squash_cleanup, + tmp_dir=self.conf.squash_tmp_dir) + self.logger.info('Image is squashed successfully') + class WorkerThread(threading.Thread): """Thread that executes tasks until the queue provides a tombstone.""" @@ -1084,7 +1081,7 @@ class KollaWorker(object): del match image = Image(image_name, canonical_name, path, parent_name=parent_name, - logger=make_a_logger(self.conf, image_name), + logger=utils.make_a_logger(self.conf, image_name), docker_client=self.dc) if self.install_type == 'source': @@ -1226,6 +1223,13 @@ def run_build(): if conf.debug: LOG.setLevel(logging.DEBUG) + if conf.squash: + squash_version = utils.get_docker_squash_version() + LOG.info('Image squash is enabled and "docker-squash" version is %s', + squash_version) + else: + LOG.info('Image squash is disabled') + kolla = KollaWorker(conf) kolla.setup_working_dir() kolla.find_dockerfiles() diff --git a/releasenotes/notes/support-docker-image-squash-8396c0de63085f5d.yaml b/releasenotes/notes/support-docker-image-squash-8396c0de63085f5d.yaml new file mode 100644 index 0000000000..2907db56fb --- /dev/null +++ b/releasenotes/notes/support-docker-image-squash-8396c0de63085f5d.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + support --squash parameter which leverage docker-squash tool to squash + newly built layers into a single new layer