IBP: Added driver and objects for building images

As far as building of OS images is nothing more than
just a stage of the whole OS installing procedure
it is sounds rational to implement this in terms
of fuel-agent. Besides, we already have plenty of utilities
which could be useful during building of images.
And some tasks are the same like pre-configuring
some files inside target OS.

Related-bug: #1433193
Implements: blueprint ibp-build-ubuntu-images
Change-Id: I3fadfb16e06e4ee16926da29b7b83ca005500698

fuel_agent/etc/fuel-agent/fuel-agent.conf.sample

@@ -17,13 +17,34 @@
 # value)
 #config_drive_path=/tmp/config-drive.img
 
+# Path where to store actual rules for udev daemon (string
+# value)
+#udev_rules_dir=/etc/udev/rules.d
+
+# Path where to store default rules for udev daemon (string
+# value)
+#udev_rules_lib_dir=/lib/udev/rules.d
+
+# Substring to which file extension .rules be renamed (string
+# value)
+#udev_rename_substr=.renamedrule
+
+# Directory where we build images (string value)
+#image_build_dir=/tmp
+
+# Directory where we build images (string value)
+#image_build_suffix=.fuel-agent-image
+
 
 #
 # Options defined in fuel_agent.cmd.agent
 #
 
-# Provision data file (string value)
-#provision_data_file=/tmp/provision.json
+# Input data file (string value)
+#input_data_file=/tmp/provision.json
+
+# Input data (json string) (string value)
+#input_data=
 
 
 #
@@ -99,6 +120,7 @@ logging_debug_format_suffix=
 # Deprecated group/name - [DEFAULT]/logfile
 log_file=/var/log/fuel-agent.log
 
+
 # (Optional) The base directory used for relative --log-file
 # paths. (string value)
 # Deprecated group/name - [DEFAULT]/logdir
@@ -120,3 +142,43 @@ log_file=/var/log/fuel-agent.log
 #syslog_log_facility=LOG_USER
 
 
+#
+# Options defined in fuel_agent.utils.artifact_utils
+#
+
+# Size of data chunk to operate with images (integer value)
+#data_chunk_size=1048576
+
+
+#
+# Options defined in fuel_agent.utils.build_utils
+#
+
+# Maximum allowed loop devices count to use (integer value)
+#max_loop_count=255
+
+# Size of sparse file in MiBs (integer value)
+#sparse_file_size=2048
+
+# System-wide major number for loop device (integer value)
+#loop_dev_major=7
+
+
+#
+# Options defined in fuel_agent.utils.utils
+#
+
+# Maximum retries count for http requests. 0 means infinite
+# (integer value)
+#http_max_retries=30
+
+# Http request timeout in seconds (floating point value)
+#http_request_timeout=10.0
+
+# Delay in seconds before the next http request retry
+# (floating point value)
+#http_retry_delay=2.0
+
+# Block size of data to read for calculating checksum (integer
+# value)
+#read_chunk_size=1048576

fuel_agent/fuel_agent/cmd/agent.py

@@ -12,30 +12,36 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import json
 import sys
 
 from oslo.config import cfg
+from oslo_serialization import jsonutils as json
 import six
 
 from fuel_agent import manager as manager
-from fuel_agent.openstack.common import log
+from fuel_agent.openstack.common import log as logging
 from fuel_agent import version
 
-opts = [
+cli_opts = [
     cfg.StrOpt(
-        'provision_data_file',
+        'input_data_file',
         default='/tmp/provision.json',
-        help='Provision data file'
+        help='Input data file'
+    ),
+    cfg.StrOpt(
+        'input_data',
+        default='',
+        help='Input data (json string)'
     ),
 ]
 
 CONF = cfg.CONF
-CONF.register_opts(opts)
+CONF.register_cli_opts(cli_opts)
 CONF(sys.argv[1:], project='fuel-agent',
      version=version.version_info.release_string())
-log.setup('fuel-agent')
-LOG = log.getLogger(__name__)
+
+logging.setup('fuel-agent')
+LOG = logging.getLogger(__name__)
 
 
 def provision():
@@ -58,6 +64,10 @@ def bootloader():
     main(['do_bootloader'])
 
 
+def build_image():
+    main(['do_build_image'])
+
+
 def print_err(line):
     sys.stderr.write(six.text_type(line))
     sys.stderr.write('\n')
@@ -72,8 +82,12 @@ def handle_exception(exc):
 
 def main(actions=None):
     try:
-        with open(CONF.provision_data_file) as f:
-            data = json.load(f)
+        if CONF.input_data:
+            data = json.loads(CONF.input_data)
+        else:
+            with open(CONF.input_data_file) as f:
+                data = json.load(f)
+        LOG.debug('Input data: %s', data)
 
         mgr = manager.Manager(data)
         if actions:
@@ -82,5 +96,6 @@ def main(actions=None):
     except Exception as exc:
         handle_exception(exc)
 
+
 if __name__ == '__main__':
     main()

fuel_agent/fuel_agent/drivers/base.py

@@ -0,0 +1,27 @@
+# Copyright 2015 Mirantis, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import abc
+import six
+
+
+@six.add_metaclass(abc.ABCMeta)
+class BaseDataDriver(object):
+    """Data driver API is to be put here.
+    For example, data validation methods,
+    methods for getting object schemes, etc.
+    """
+
+    def __init__(self, data):
+        self.data = data

fuel_agent/fuel_agent/drivers/nailgun.py

@@ -12,10 +12,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import itertools
 import math
 import os
 import six
+import yaml
 
+from six.moves.urllib.parse import urljoin
+from six.moves.urllib.parse import urlparse
+from six.moves.urllib.parse import urlsplit
+
+from fuel_agent.drivers.base import BaseDataDriver
 from fuel_agent.drivers import ks_spaces_validator
 from fuel_agent import errors
 from fuel_agent import objects
@@ -23,9 +30,6 @@ from fuel_agent.openstack.common import log as logging
 from fuel_agent.utils import hardware_utils as hu
 from fuel_agent.utils import utils
 
-from six.moves.urllib.parse import urljoin
-from six.moves.urllib.parse import urlparse
-import yaml
 
 LOG = logging.getLogger(__name__)
 
@@ -60,11 +64,9 @@ def match_device(hu_disk, ks_disk):
     return False
 
 
-class Nailgun(object):
+class Nailgun(BaseDataDriver):
     def __init__(self, data):
-        # Here data is expected to be raw provisioning data
-        # how it is given by nailgun
-        self.data = data
+        super(Nailgun, self).__init__(data)
 
         # this var is used as a flag that /boot fs
         # has already been added. we need this to
@@ -394,16 +396,13 @@ class Nailgun(object):
         filename = os.path.basename(urlparse(root_uri).path).split('.')[0] + \
             '.yaml'
         metadata_url = urljoin(root_uri, filename)
-        image_meta = {}
-        raw_image_meta = None
         try:
-            raw_image_meta = yaml.load(
+            image_meta = yaml.load(
                 utils.init_http_request(metadata_url).text)
         except Exception as e:
             LOG.exception(e)
             LOG.debug('Failed to fetch/decode image meta data')
-        if raw_image_meta:
-            [image_meta.update(img_info) for img_info in raw_image_meta]
+            image_meta = {}
         # We assume for every file system user may provide a separate
         # file system image. For example if partitioning scheme has
         # /, /boot, /var/lib file systems then we will try to get images
@@ -415,13 +414,120 @@ class Nailgun(object):
             LOG.debug('Adding image for fs %s: uri=%s format=%s container=%s' %
                       (mount_point, image_data['uri'],
                        image_data['format'], image_data['container']))
+            iname = os.path.basename(urlparse(image_data['uri']).path)
+            imeta = next(itertools.chain(
+                (img for img in image_meta.get('images', [])
+                 if img['container_name'] == iname), [{}]))
             image_scheme.add_image(
                 uri=image_data['uri'],
                 target_device=self.partition_scheme.fs_by_mount(
                     mount_point).device,
                 format=image_data['format'],
                 container=image_data['container'],
-                size=image_meta.get(mount_point, {}).get('size'),
-                md5=image_meta.get(mount_point, {}).get('md5'),
+                size=imeta.get('raw_size'),
+                md5=imeta.get('raw_md5'),
             )
         return image_scheme
+
+
+class NailgunBuildImage(BaseDataDriver):
+
+    # TODO(kozhukalov):
+    # This list of packages is used by default only if another
+    # list isn't given in build image data. In the future
+    # we need to handle package list in nailgun. Even more,
+    # in the future, we'll be building not only ubuntu images
+    # and we'll likely move this list into some kind of config.
+    DEFAULT_TRUSTY_PACKAGES = [
+        "acl",
+        "anacron",
+        "bash-completion",
+        "bridge-utils",
+        "bsdmainutils",
+        "build-essential",
+        "cloud-init",
+        "curl",
+        "daemonize",
+        "debconf-utils",
+        "gdisk",
+        "grub-pc",
+        "linux-firmware",
+        "linux-firmware-nonfree",
+        "linux-headers-generic-lts-trusty",
+        "linux-image-generic-lts-trusty",
+        "lvm2",
+        "mcollective",
+        "mdadm",
+        "nailgun-agent",
+        "nailgun-mcagents",
+        "nailgun-net-check",
+        "ntp",
+        "openssh-client",
+        "openssh-server",
+        "puppet",
+        "python-amqp",
+        "ruby-augeas",
+        "ruby-ipaddress",
+        "ruby-json",
+        "ruby-netaddr",
+        "ruby-openstack",
+        "ruby-shadow",
+        "ruby-stomp",
+        "telnet",
+        "ubuntu-minimal",
+        "ubuntu-standard",
+        "uuid-runtime",
+        "vim",
+        "virt-what",
+        "vlan",
+    ]
+
+    def __init__(self, data):
+        super(NailgunBuildImage, self).__init__(data)
+        self.parse_schemes()
+        self.parse_operating_system()
+
+    def parse_operating_system(self):
+        if self.data.get('codename').lower() != 'trusty':
+            raise errors.WrongInputDataError(
+                'Currently, only Ubuntu Trusty is supported, given '
+                'codename is {0}'.format(self.data.get('codename')))
+
+        packages = self.data.get('packages', self.DEFAULT_TRUSTY_PACKAGES)
+
+        repos = []
+        for repo in self.data['repos']:
+            repos.append(objects.DEBRepo(
+                name=repo['name'],
+                uri=repo['uri'],
+                suite=repo['suite'],
+                section=repo['section'],
+                priority=repo['priority']))
+
+        self.operating_system = objects.Ubuntu(repos=repos, packages=packages)
+
+    def parse_schemes(self):
+        self.image_scheme = objects.ImageScheme()
+        self.partition_scheme = objects.PartitionScheme()
+
+        for mount, image in six.iteritems(self.data['image_data']):
+            filename = os.path.basename(urlsplit(image['uri']).path)
+            # Loop does not allocate any loop device
+            # during initialization.
+            device = objects.Loop()
+
+            self.image_scheme.add_image(
+                uri='file://' + os.path.join(self.data['output'], filename),
+                format=image['format'],
+                container=image['container'],
+                target_device=device)
+
+            self.partition_scheme.add_fs(
+                device=device,
+                mount=mount,
+                fs_type=image['format'])
+
+            if mount == '/':
+                metadata_filename = filename.split('.', 1)[0] + '.yaml'
+                self.metadata_uri = 'file://' + os.path.join(
+                    self.data['output'], metadata_filename)

fuel_agent/fuel_agent/errors.py

@@ -19,6 +19,10 @@ class BaseError(Exception):
         super(BaseError, self).__init__(message, *args, **kwargs)
 
 
+class WrongInputDataError(BaseError):
+    pass
+
+
 class WrongPartitionSchemeError(BaseError):
     pass
 
@@ -146,3 +150,11 @@ class ImageChecksumMismatchError(BaseError):
 
 class NoFreeLoopDevices(BaseError):
     pass
+
+
+class WrongRepositoryError(BaseError):
+    pass
+
+
+class WrongDeviceError(BaseError):
+    pass

fuel_agent/fuel_agent/manager.py

@@ -13,12 +13,18 @@
 # limitations under the License.
 
 import os
+import shutil
+import signal
+import tempfile
+import time
+import yaml
 
 from oslo.config import cfg
 
 from fuel_agent import errors
 from fuel_agent.openstack.common import log as logging
 from fuel_agent.utils import artifact_utils as au
+from fuel_agent.utils import build_utils as bu
 from fuel_agent.utils import fs_utils as fu
 from fuel_agent.utils import grub_utils as gu
 from fuel_agent.utils import lvm_utils as lu
@@ -27,11 +33,6 @@ from fuel_agent.utils import partition_utils as pu
 from fuel_agent.utils import utils
 
 opts = [
-    cfg.StrOpt(
-        'data_driver',
-        default='nailgun',
-        help='Data driver'
-    ),
     cfg.StrOpt(
         'nc_template_path',
         default='/usr/share/fuel-agent/cloud-init-templates',
@@ -67,10 +68,29 @@ opts = [
         default='empty_rule',
         help='Correct empty rule for udev daemon',
     ),
+    cfg.StrOpt(
+        'image_build_dir',
+        default='/tmp',
+        help='Directory where the image is supposed to be built',
+    ),
+    cfg.StrOpt(
+        'image_build_suffix',
+        default='.fuel-agent-image',
+        help='Suffix which is used while creating temporary files',
+    ),
+]
+
+cli_opts = [
+    cfg.StrOpt(
+        'data_driver',
+        default='nailgun',
+        help='Data driver'
+    ),
 ]
 
 CONF = cfg.CONF
 CONF.register_opts(opts)
+CONF.register_cli_opts(cli_opts)
 
 LOG = logging.getLogger(__name__)
 
@@ -311,41 +331,49 @@ class Manager(object):
                           (image.format, image.target_device))
                 fu.extend_fs(image.format, image.target_device)
 
-    def mount_target(self, chroot):
+    # TODO(kozhukalov): write tests
+    def mount_target(self, chroot, treat_mtab=True, pseudo=True):
+        """Mount a set of file systems into a chroot
+
+        :param chroot: Directory where to mount file systems
+        :param treat_mtab: If mtab needs to be actualized (Default: True)
+        :param pseudo: If pseudo file systems
+        need to be mounted (Default: True)
+        """
         LOG.debug('Mounting target file systems')
         # Here we are going to mount all file systems in partition scheme.
-        # Shorter paths earlier. We sort all mount points by their depth.
-        # ['/', '/boot', '/var', '/var/lib/mysql']
-        key = lambda x: len(x.mount.rstrip('/').split('/'))
-        for fs in sorted(self.driver.partition_scheme.fss, key=key):
+        for fs in self.driver.partition_scheme.fs_sorted_by_depth():
             if fs.mount == 'swap':
                 continue
             mount = chroot + fs.mount
-            if not os.path.isdir(mount):
-                os.makedirs(mount, mode=0o755)
-            fu.mount_fs(fs.type, fs.device, mount)
-        fu.mount_bind(chroot, '/sys')
-        fu.mount_bind(chroot, '/dev')
-        fu.mount_bind(chroot, '/proc')
-        mtab = utils.execute(
-            'chroot', chroot, 'grep', '-v', 'rootfs', '/proc/mounts')[0]
-        mtab_path = chroot + '/etc/mtab'
-        if os.path.islink(mtab_path):
-            os.remove(mtab_path)
-        with open(mtab_path, 'wb') as f:
-            f.write(mtab)
+            utils.makedirs_if_not_exists(mount)
+            fu.mount_fs(fs.type, str(fs.device), mount)
 
-    def umount_target(self, chroot):
+        if pseudo:
+            for path in ('/sys', '/dev', '/proc'):
+                utils.makedirs_if_not_exists(chroot + path)
+                fu.mount_bind(chroot, path)
+
+        if treat_mtab:
+            mtab = utils.execute(
+                'chroot', chroot, 'grep', '-v', 'rootfs', '/proc/mounts')[0]
+            mtab_path = chroot + '/etc/mtab'
+            if os.path.islink(mtab_path):
+                os.remove(mtab_path)
+            with open(mtab_path, 'wb') as f:
+                f.write(mtab)
+
+    # TODO(kozhukalov): write tests
+    def umount_target(self, chroot, pseudo=True):
         LOG.debug('Umounting target file systems')
-        fu.umount_fs(chroot + '/proc')
-        fu.umount_fs(chroot + '/dev')
-        fu.umount_fs(chroot + '/sys')
-        key = lambda x: len(x.mount.rstrip('/').split('/'))
-        for fs in sorted(self.driver.partition_scheme.fss,
-                         key=key, reverse=True):
+        if pseudo:
+            for path in ('/proc', '/dev', '/sys'):
+                fu.umount_fs(chroot + path)
+        for fs in self.driver.partition_scheme.fs_sorted_by_depth(
+                reverse=True):
             if fs.mount == 'swap':
                 continue
-            fu.umount_fs(fs.device)
+            fu.umount_fs(chroot + fs.mount)
 
     def do_bootloader(self):
         LOG.debug('--- Installing bootloader (do_bootloader) ---')
@@ -416,3 +444,200 @@ class Manager(object):
         self.do_configdrive()
         self.do_copyimage()
         self.do_bootloader()
+        LOG.debug('--- Provisioning END (do_provisioning) ---')
+
+    # TODO(kozhukalov): Split this huge method
+    # into a set of smaller ones
+    # https://bugs.launchpad.net/fuel/+bug/1444090
+    def do_build_image(self):
+        """Building OS images includes the following steps
+        1) create temporary sparse files for all images (truncate)
+        2) attach temporary files to loop devices (losetup)
+        3) create file systems on these loop devices
+        4) create temporary chroot directory
+        5) mount loop devices into chroot directory
+        6) install operating system (debootstrap and apt-get)
+        7) configure OS (clean sources.list and preferences, etc.)
+        8) umount loop devices
+        9) resize file systems on loop devices
+        10) shrink temporary sparse files (images)
+        11) containerize (gzip) temporary sparse files
+        12) move temporary gzipped files to their final location
+        """
+        LOG.info('--- Building image (do_build_image) ---')
+        # TODO(kozhukalov): Implement metadata
+        # as a pluggable data driver to avoid any fixed format.
+        metadata = {}
+
+        # TODO(kozhukalov): implement this using image metadata
+        # we need to compare list of packages and repos
+        LOG.info('*** Checking if image exists ***')
+        if all([os.path.exists(img.uri.split('file://', 1)[1])
+                for img in self.driver.image_scheme.images]):
+            LOG.debug('All necessary images are available. '
+                      'Nothing needs to be done.')
+            return
+        LOG.debug('At least one of the necessary images is unavailable. '
+                  'Starting build process.')
+
+        LOG.info('*** Preparing image space ***')
+        for image in self.driver.image_scheme.images:
+            LOG.debug('Creating temporary sparsed file for the '
+                      'image: %s', image.uri)
+            img_tmp_file = bu.create_sparse_tmp_file(
+                dir=CONF.image_build_dir, suffix=CONF.image_build_suffix)
+            LOG.debug('Temporary file: %s', img_tmp_file)
+
+            # we need to remember those files
+            # to be able to shrink them and move in the end
+            image.img_tmp_file = img_tmp_file
+
+            LOG.debug('Looking for a free loop device')
+            image.target_device.name = bu.get_free_loop_device()
+
+            LOG.debug('Attaching temporary image file to free loop device')
+            bu.attach_file_to_loop(img_tmp_file, str(image.target_device))
+
+            # find fs with the same loop device object
+            # as image.target_device
+            fs = self.driver.partition_scheme.fs_by_device(image.target_device)
+
+            LOG.debug('Creating file system on the image')
+            fu.make_fs(
+                fs_type=fs.type,
+                fs_options=fs.options,
+                fs_label=fs.label,
+                dev=str(fs.device))
+
+        LOG.debug('Creating temporary chroot directory')
+        chroot = tempfile.mkdtemp(
+            dir=CONF.image_build_dir, suffix=CONF.image_build_suffix)
+        LOG.debug('Temporary chroot: %s', chroot)
+
+        # mounting all images into chroot tree
+        self.mount_target(chroot, treat_mtab=False, pseudo=False)
+
+        LOG.info('*** Shipping image content ***')
+        LOG.debug('Installing operating system into image')
+        # FIXME(kozhukalov): !!! we need this part to be OS agnostic
+
+        # DEBOOTSTRAP
+        # we use first repo as the main mirror
+        uri = self.driver.operating_system.repos[0].uri
+        suite = self.driver.operating_system.repos[0].suite
+
+        LOG.debug('Preventing services from being get started')
+        bu.suppress_services_start(chroot)
+        LOG.debug('Installing base operating system using debootstrap')
+        bu.run_debootstrap(uri=uri, suite=suite, chroot=chroot)
+
+        # APT-GET
+        LOG.debug('Configuring apt inside chroot')
+        LOG.debug('Setting environment variables')
+        bu.set_apt_get_env()
+        LOG.debug('Allowing unauthenticated repos')
+        bu.pre_apt_get(chroot)
+
+        for repo in self.driver.operating_system.repos:
+            LOG.debug('Adding repository source: name={name}, uri={uri},'
+                      'suite={suite}, section={section}'.format(
+                          name=repo.name, uri=repo.uri,
+                          suite=repo.suite, section=repo.section))
+            bu.add_apt_source(
+                name=repo.name,
+                uri=repo.uri,
+                suite=repo.suite,
+                section=repo.section,
+                chroot=chroot)
+            LOG.debug('Adding repository preference: '
+                      'name={name}, priority={priority}'.format(
+                          name=repo.name, priority=repo.priority))
+            bu.add_apt_preference(
+                name=repo.name,
+                priority=repo.priority,
+                suite=repo.suite,
+                section=repo.section,
+                chroot=chroot)
+
+            metadata.setdefault('repos', []).append({
+                'type': 'deb',
+                'name': repo.name,
+                'uri': repo.uri,
+                'suite': repo.suite,
+                'section': repo.section,
+                'priority': repo.priority,
+                'meta': repo.meta})
+
+        LOG.debug('Preventing services from being get started')
+        bu.suppress_services_start(chroot)
+
+        packages = self.driver.operating_system.packages
+        metadata['packages'] = packages
+
+        # we need /proc to be mounted for apt-get success
+        proc_path = os.path.join(chroot, 'proc')
+        utils.makedirs_if_not_exists(proc_path)
+        fu.mount_bind(chroot, '/proc')
+
+        LOG.debug('Installing packages using apt-get: %s',
+                  ' '.join(packages))
+        bu.run_apt_get(chroot, packages=packages)
+
+        LOG.debug('Post-install OS configuration')
+        bu.do_post_inst(chroot)
+
+        LOG.debug('Making sure there are no running processes '
+                  'inside chroot before trying to umount chroot')
+        bu.send_signal_to_chrooted_processes(chroot, signal.SIGTERM)
+        # We assume there might be some processes which
+        # require some reasonable time to stop before we try
+        # to send them SIGKILL. Waiting for 2 seconds
+        # looks reasonable here.
+        time.sleep(2)
+        bu.send_signal_to_chrooted_processes(chroot, signal.SIGKILL)
+
+        LOG.info('*** Finalizing image space ***')
+        fu.umount_fs(proc_path)
+        # umounting all loop devices
+        self.umount_target(chroot, pseudo=False)
+
+        for image in self.driver.image_scheme.images:
+            LOG.debug('Deattaching loop device from file: %s',
+                      image.img_tmp_file)
+            bu.deattach_loop(str(image.target_device))
+            LOG.debug('Shrinking temporary image file: %s',
+                      image.img_tmp_file)
+            bu.shrink_sparse_file(image.img_tmp_file)
+
+            raw_size = os.path.getsize(image.img_tmp_file)
+            raw_md5 = utils.calculate_md5(image.img_tmp_file, raw_size)
+
+            LOG.debug('Containerizing temporary image file: %s',
+                      image.img_tmp_file)
+            img_tmp_containerized = bu.containerize(
+                image.img_tmp_file, image.container)
+            img_containerized = image.uri.split('file://', 1)[1]
+
+            # NOTE(kozhukalov): implement abstract publisher
+            LOG.debug('Moving image file to the final location: %s',
+                      img_containerized)
+            shutil.move(img_tmp_containerized, img_containerized)
+
+            container_size = os.path.getsize(img_containerized)
+            container_md5 = utils.calculate_md5(
+                img_containerized, container_size)
+            metadata.setdefault('images', []).append({
+                'raw_md5': raw_md5,
+                'raw_size': raw_size,
+                'raw_name': None,
+                'container_name': os.path.basename(img_containerized),
+                'container_md5': container_md5,
+                'container_size': container_size,
+                'container': image.container,
+                'format': image.format})
+
+        # NOTE(kozhukalov): implement abstract publisher
+        LOG.debug('Image metadata: %s', metadata)
+        with open(self.driver.metadata_uri.split('file://', 1)[1], 'w') as f:
+            yaml.safe_dump(metadata, stream=f)
+        LOG.info('--- Building image END (do_build_image) ---')

fuel_agent/fuel_agent/objects/__init__.py

@@ -16,8 +16,11 @@ from fuel_agent.objects.configdrive import ConfigDriveCommon
 from fuel_agent.objects.configdrive import ConfigDriveMcollective
 from fuel_agent.objects.configdrive import ConfigDrivePuppet
 from fuel_agent.objects.configdrive import ConfigDriveScheme
+from fuel_agent.objects.device import Loop
 from fuel_agent.objects.image import Image
 from fuel_agent.objects.image import ImageScheme
+from fuel_agent.objects.operating_system import OperatingSystem
+from fuel_agent.objects.operating_system import Ubuntu
 from fuel_agent.objects.partition import Fs
 from fuel_agent.objects.partition import Lv
 from fuel_agent.objects.partition import Md
@@ -25,9 +28,14 @@ from fuel_agent.objects.partition import Partition
 from fuel_agent.objects.partition import PartitionScheme
 from fuel_agent.objects.partition import Pv
 from fuel_agent.objects.partition import Vg
+from fuel_agent.objects.repo import DEBRepo
+from fuel_agent.objects.repo import Repo
 
 __all__ = [
     'Partition', 'Pv', 'Vg', 'Lv', 'Md', 'Fs', 'PartitionScheme',
     'ConfigDriveCommon', 'ConfigDrivePuppet', 'ConfigDriveMcollective',
     'ConfigDriveScheme', 'Image', 'ImageScheme',
+    'OperatingSystem', 'Ubuntu',
+    'Repo', 'DEBRepo',
+    'Loop',
 ]

fuel_agent/fuel_agent/objects/device.py

@@ -0,0 +1,28 @@
+# Copyright 2015 Mirantis, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from fuel_agent import errors
+
+
+class Loop(object):
+    def __init__(self, name=None):
+        self.name = name
+
+    def __str__(self):
+        if self.name:
+            return self.name
+        raise errors.WrongDeviceError(
+            'Loop device can not be stringified. '
+            'Name attribute is not set. Current: '
+            'name={0}'.format(self.name))

fuel_agent/fuel_agent/objects/operating_system.py

@@ -0,0 +1,23 @@
+# Copyright 2015 Mirantis, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class OperatingSystem(object):
+    def __init__(self, repos, packages):
+        self.repos = repos
+        self.packages = packages
+
+
+class Ubuntu(OperatingSystem):
+    pass

fuel_agent/fuel_agent/objects/partition.py

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
+
 from fuel_agent import errors
 from fuel_agent.openstack.common import log as logging
 
@@ -289,6 +291,15 @@ class PartitionScheme(object):
         if found:
             return found[0]
 
+    def fs_sorted_by_depth(self, reverse=False):
+        """Getting file systems sorted by path length. Shorter paths earlier.
+        ['/', '/boot', '/var', '/var/lib/mysql']
+        :param reverse: Sort backward (Default: False)
+        """
+        def key(x):
+            return x.mount.rstrip(os.path.sep).count(os.path.sep)
+        return sorted(self.fss, key=key, reverse=reverse)
+
     def lv_by_device_name(self, device_name):
         found = filter(lambda x: x.device_name == device_name, self.lvs)
         if found:

fuel_agent/fuel_agent/objects/repo.py

@@ -0,0 +1,28 @@
+# Copyright 2015 Mirantis, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class Repo(object):
+    def __init__(self, name, uri, priority=None):
+        self.name = name
+        self.uri = uri
+        self.priority = priority
+
+
+class DEBRepo(Repo):
+    def __init__(self, name, uri, suite, section, meta=None, priority=None):
+        super(DEBRepo, self).__init__(name, uri, priority)
+        self.suite = suite
+        self.section = section
+        self.meta = meta

fuel_agent/fuel_agent/tests/test_manager.py

@@ -14,12 +14,14 @@
 
 import mock
 import os
+import signal
 
 from oslo.config import cfg
 from oslotest import base as test_base
 
 from fuel_agent import errors
 from fuel_agent import manager
+from fuel_agent import objects
 from fuel_agent.objects import partition
 from fuel_agent.tests import test_nailgun
 from fuel_agent.utils import artifact_utils as au
@@ -328,3 +330,183 @@ class TestManager(test_base.BaseTestCase):
         self.assertEqual(2, len(self.mgr.driver.image_scheme.images))
         self.assertRaises(errors.ImageChecksumMismatchError,
                           self.mgr.do_copyimage)
+
+    @mock.patch('fuel_agent.manager.bu', create=True)
+    @mock.patch('fuel_agent.manager.fu', create=True)
+    @mock.patch('fuel_agent.manager.utils', create=True)
+    @mock.patch('fuel_agent.manager.os', create=True)
+    @mock.patch('fuel_agent.manager.shutil.move')
+    @mock.patch('fuel_agent.manager.open',
+                create=True, new_callable=mock.mock_open)
+    @mock.patch('fuel_agent.manager.tempfile.mkdtemp')
+    @mock.patch('fuel_agent.manager.time.sleep')
+    @mock.patch('fuel_agent.manager.yaml.safe_dump')
+    @mock.patch.object(manager.Manager, 'mount_target')
+    @mock.patch.object(manager.Manager, 'umount_target')
+    def test_do_build_image(self, mock_umount_target, mock_mount_target,
+                            mock_yaml_dump, mock_sleep, mock_mkdtemp,
+                            mock_open, mock_shutil_move, mock_os,
+                            mock_utils, mock_fu, mock_bu):
+
+        loops = [objects.Loop(), objects.Loop()]
+
+        self.mgr.driver.image_scheme = objects.ImageScheme([
+            objects.Image('file:///fake/img.img.gz', loops[0], 'ext4', 'gzip'),
+            objects.Image('file:///fake/img-boot.img.gz',
+                          loops[1], 'ext2', 'gzip')])
+        self.mgr.driver.partition_scheme = objects.PartitionScheme()
+        self.mgr.driver.partition_scheme.add_fs(
+            device=loops[0], mount='/', fs_type='ext4')
+        self.mgr.driver.partition_scheme.add_fs(
+            device=loops[1], mount='/boot', fs_type='ext2')
+        self.mgr.driver.metadata_uri = 'file:///fake/img.yaml'
+        self.mgr.driver.operating_system = objects.Ubuntu(
+            repos=[
+                objects.DEBRepo('ubuntu', 'http://fakeubuntu',
+                                'trusty', 'fakesection'),
+                objects.DEBRepo('mos', 'http://fakemos',
+                                'mosX.Y', 'fakesection', priority=1000)],
+            packages=['fakepackage1', 'fakepackage2'])
+
+        mock_os.path.exists.return_value = False
+        mock_os.path.join.return_value = '/tmp/imgdir/proc'
+        mock_os.path.basename.side_effect = ['img.img.gz', 'img-boot.img.gz']
+        mock_bu.create_sparse_tmp_file.side_effect = \
+            ['/tmp/img', '/tmp/img-boot']
+        mock_bu.get_free_loop_device.side_effect = ['/dev/loop0', '/dev/loop1']
+        mock_mkdtemp.return_value = '/tmp/imgdir'
+        getsize_side = [20, 2, 10, 1]
+        mock_os.path.getsize.side_effect = getsize_side
+        md5_side = ['fakemd5_raw', 'fakemd5_gzip',
+                    'fakemd5_raw_boot', 'fakemd5_gzip_boot']
+        mock_utils.calculate_md5.side_effect = md5_side
+        mock_bu.containerize.side_effect = ['/tmp/img.gz', '/tmp/img-boot.gz']
+
+        self.mgr.do_build_image()
+        self.assertEqual(
+            [mock.call('/fake/img.img.gz'),
+             mock.call('/fake/img-boot.img.gz')],
+            mock_os.path.exists.call_args_list)
+        self.assertEqual([mock.call(dir=CONF.image_build_dir,
+                                    suffix=CONF.image_build_suffix)] * 2,
+                         mock_bu.create_sparse_tmp_file.call_args_list)
+        self.assertEqual([mock.call()] * 2,
+                         mock_bu.get_free_loop_device.call_args_list)
+        self.assertEqual([mock.call('/tmp/img', '/dev/loop0'),
+                          mock.call('/tmp/img-boot', '/dev/loop1')],
+                         mock_bu.attach_file_to_loop.call_args_list)
+        self.assertEqual([mock.call(fs_type='ext4', fs_options='',
+                                    fs_label='', dev='/dev/loop0'),
+                          mock.call(fs_type='ext2', fs_options='',
+                                    fs_label='', dev='/dev/loop1')],
+                         mock_fu.make_fs.call_args_list)
+        mock_mkdtemp.assert_called_once_with(dir=CONF.image_build_dir,
+                                             suffix=CONF.image_build_suffix)
+        mock_mount_target.assert_called_once_with(
+            '/tmp/imgdir', treat_mtab=False, pseudo=False)
+        self.assertEqual([mock.call('/tmp/imgdir')] * 2,
+                         mock_bu.suppress_services_start.call_args_list)
+        mock_bu.run_debootstrap.assert_called_once_with(
+            uri='http://fakeubuntu', suite='trusty', chroot='/tmp/imgdir')
+        mock_bu.set_apt_get_env.assert_called_once_with()
+        mock_bu.pre_apt_get.assert_called_once_with('/tmp/imgdir')
+        self.assertEqual([
+            mock.call(name='ubuntu',
+                      uri='http://fakeubuntu',
+                      suite='trusty',
+                      section='fakesection',
+                      chroot='/tmp/imgdir'),
+            mock.call(name='mos',
+                      uri='http://fakemos',
+                      suite='mosX.Y',
+                      section='fakesection',
+                      chroot='/tmp/imgdir')],
+            mock_bu.add_apt_source.call_args_list)
+        self.assertEqual([
+            mock.call(name='ubuntu',
+                      priority=None,
+                      suite='trusty',
+                      section='fakesection',
+                      chroot='/tmp/imgdir'),
+            mock.call(name='mos',
+                      priority=1000,
+                      suite='mosX.Y',
+                      section='fakesection',
+                      chroot='/tmp/imgdir')],
+            mock_bu.add_apt_preference.call_args_list)
+        mock_utils.makedirs_if_not_exists.assert_called_once_with(
+            '/tmp/imgdir/proc')
+        mock_fu.mount_bind.assert_called_once_with('/tmp/imgdir', '/proc')
+        mock_bu.run_apt_get.assert_called_once_with(
+            '/tmp/imgdir', packages=['fakepackage1', 'fakepackage2'])
+        mock_bu.do_post_inst.assert_called_once_with('/tmp/imgdir')
+        signal_calls = mock_bu.send_signal_to_chrooted_processes.call_args_list
+        self.assertEqual([mock.call('/tmp/imgdir', signal.SIGTERM),
+                          mock.call('/tmp/imgdir', signal.SIGKILL)],
+                         signal_calls)
+        mock_sleep.assert_called_once_with(2)
+        mock_fu.umount_fs.assert_called_once_with('/tmp/imgdir/proc')
+        mock_umount_target.assert_called_once_with('/tmp/imgdir', pseudo=False)
+        self.assertEqual([mock.call('/dev/loop0'), mock.call('/dev/loop1')],
+                         mock_bu.deattach_loop.call_args_list)
+        self.assertEqual([mock.call('/tmp/img'), mock.call('/tmp/img-boot')],
+                         mock_bu.shrink_sparse_file.call_args_list)
+        self.assertEqual([mock.call('/tmp/img'),
+                          mock.call('/fake/img.img.gz'),
+                          mock.call('/tmp/img-boot'),
+                          mock.call('/fake/img-boot.img.gz')],
+                         mock_os.path.getsize.call_args_list)
+        self.assertEqual([mock.call('/tmp/img', 20),
+                          mock.call('/fake/img.img.gz', 2),
+                          mock.call('/tmp/img-boot', 10),
+                          mock.call('/fake/img-boot.img.gz', 1)],
+                         mock_utils.calculate_md5.call_args_list)
+        self.assertEqual([mock.call('/tmp/img', 'gzip'),
+                          mock.call('/tmp/img-boot', 'gzip')],
+                         mock_bu.containerize.call_args_list)
+        mock_open.assert_called_once_with('/fake/img.yaml', 'w')
+        self.assertEqual(
+            [mock.call('/tmp/img.gz', '/fake/img.img.gz'),
+             mock.call('/tmp/img-boot.gz', '/fake/img-boot.img.gz')],
+            mock_shutil_move.call_args_list)
+
+        metadata = {}
+        for repo in self.mgr.driver.operating_system.repos:
+            metadata.setdefault('repos', []).append({
+                'type': 'deb',
+                'name': repo.name,
+                'uri': repo.uri,
+                'suite': repo.suite,
+                'section': repo.section,
+                'priority': repo.priority,
+                'meta': repo.meta})
+        metadata['packages'] = self.mgr.driver.operating_system.packages
+        metadata['images'] = [
+            {
+                'raw_md5': md5_side[0],
+                'raw_size': getsize_side[0],
+                'raw_name': None,
+                'container_name':
+                os.path.basename(
+                    self.mgr.driver.image_scheme.images[0].uri.split(
+                        'file://', 1)[1]),
+                'container_md5': md5_side[1],
+                'container_size': getsize_side[1],
+                'container': self.mgr.driver.image_scheme.images[0].container,
+                'format': self.mgr.driver.image_scheme.images[0].format
+            },
+            {
+                'raw_md5': md5_side[2],
+                'raw_size': getsize_side[2],
+                'raw_name': None,
+                'container_name':
+                os.path.basename(
+                    self.mgr.driver.image_scheme.images[1].uri.split(
+                        'file://', 1)[1]),
+                'container_md5': md5_side[3],
+                'container_size': getsize_side[3],
+                'container': self.mgr.driver.image_scheme.images[1].container,
+                'format': self.mgr.driver.image_scheme.images[1].format
+            }
+        ]
+        mock_yaml_dump.assert_called_once_with(metadata, stream=mock_open())

fuel_agent/fuel_agent/tests/test_nailgun.py

@@ -616,7 +616,8 @@ class TestNailgun(test_base.BaseTestCase):
     @mock.patch.object(utils, 'init_http_request')
     @mock.patch.object(hu, 'list_block_devices')
     def test_image_scheme_with_checksums(self, mock_lbd, mock_http_req):
-        fake_image_meta = [{'/': {'md5': 'fakeroot', 'size': 1}}]
+        fake_image_meta = {'images': [{'raw_md5': 'fakeroot', 'raw_size': 1,
+                                       'container_name': 'fake_image.img.gz'}]}
         prop_mock = mock.PropertyMock(return_value=yaml.dump(fake_image_meta))
         type(mock_http_req.return_value).text = prop_mock
         mock_lbd.return_value = LIST_BLOCK_DEVICES_SAMPLE
@@ -646,8 +647,9 @@ class TestNailgun(test_base.BaseTestCase):
                              expected_images[i].format)
             self.assertEqual(img.container,
                              expected_images[i].container)
-            self.assertEqual(img.size, fake_image_meta[0]['/']['size'])
-            self.assertEqual(img.md5, fake_image_meta[0]['/']['md5'])
+            self.assertEqual(
+                img.size, fake_image_meta['images'][0]['raw_size'])
+            self.assertEqual(img.md5, fake_image_meta['images'][0]['raw_md5'])
 
     def test_getlabel(self):
         self.assertEqual('', self.drv._getlabel(None))

fuel_agent/fuel_agent/tests/test_nailgun_build_image.py

@@ -0,0 +1,244 @@
+# Copyright 2015 Mirantis, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mock
+import os
+import six
+from six.moves.urllib.parse import urlsplit
+
+from oslotest import base as test_base
+
+from fuel_agent.drivers.nailgun import NailgunBuildImage
+from fuel_agent import errors
+from fuel_agent import objects
+
+DEFAULT_TRUSTY_PACKAGES = [
+    "acl",
+    "anacron",
+    "bash-completion",
+    "bridge-utils",
+    "bsdmainutils",
+    "build-essential",
+    "cloud-init",
+    "curl",
+    "daemonize",
+    "debconf-utils",
+    "gdisk",
+    "grub-pc",
+    "linux-firmware",
+    "linux-firmware-nonfree",
+    "linux-headers-generic-lts-trusty",
+    "linux-image-generic-lts-trusty",
+    "lvm2",
+    "mcollective",
+    "mdadm",
+    "nailgun-agent",
+    "nailgun-mcagents",
+    "nailgun-net-check",
+    "ntp",
+    "openssh-client",
+    "openssh-server",
+    "puppet",
+    "python-amqp",
+    "ruby-augeas",
+    "ruby-ipaddress",
+    "ruby-json",
+    "ruby-netaddr",
+    "ruby-openstack",
+    "ruby-shadow",
+    "ruby-stomp",
+    "telnet",
+    "ubuntu-minimal",
+    "ubuntu-standard",
+    "uuid-runtime",
+    "vim",
+    "virt-what",
+    "vlan",
+]
+
+REPOS_SAMPLE = [
+    {
+        "name": "ubuntu",
+        "section": "main universe multiverse",
+        "uri": "http://archive.ubuntu.com/ubuntu/",
+        "priority": None,
+        "suite": "trusty",
+        "type": "deb"
+    },
+    {
+        "name": "mos",
+        "section": "main restricted",
+        "uri": "http://10.20.0.2:8080/2014.2-6.1/ubuntu/x86_64",
+        "priority": 1050,
+        "suite": "mos6.1",
+        "type": "deb"
+    }
+]
+
+IMAGE_DATA_SAMPLE = {
+    "/boot": {
+        "container": "gzip",
+        "uri": "http://10.20.0.2:8080/path/to/img-boot.img.gz",
+        "format": "ext2"
+    },
+    "/": {
+        "container": "gzip",
+        "uri": "http://10.20.0.2:8080/path/to/img.img.gz",
+        "format": "ext4"
+    }
+}
+
+
+class TestNailgunBuildImage(test_base.BaseTestCase):
+
+    def test_default_trusty_packages(self):
+        self.assertEqual(NailgunBuildImage.DEFAULT_TRUSTY_PACKAGES,
+                         DEFAULT_TRUSTY_PACKAGES)
+
+    @mock.patch.object(NailgunBuildImage, '__init__')
+    def test_parse_operating_system_error_bad_codename(self, mock_init):
+        mock_init.return_value = None
+        driver = NailgunBuildImage()
+        driver.data = {'codename': 'not-trusty'}
+        self.assertRaises(errors.WrongInputDataError,
+                          driver.parse_operating_system)
+
+    @mock.patch('fuel_agent.objects.Ubuntu')
+    @mock.patch.object(NailgunBuildImage, '__init__')
+    def test_parse_operating_system_packages_given(self, mock_init, mock_ub):
+        mock_init.return_value = None
+        data = {
+            'repos': [],
+            'codename': 'trusty',
+            'packages': ['pack']
+        }
+        driver = NailgunBuildImage()
+        driver.data = data
+        mock_ub_instance = mock_ub.return_value
+        mock_ub_instance.packages = data['packages']
+        driver.parse_operating_system()
+        mock_ub.assert_called_once_with(repos=[], packages=data['packages'])
+        self.assertEqual(driver.operating_system.packages, data['packages'])
+
+    @mock.patch('fuel_agent.objects.Ubuntu')
+    @mock.patch.object(NailgunBuildImage, '__init__')
+    def test_parse_operating_system_packages_not_given(
+            self, mock_init, mock_ub):
+        mock_init.return_value = None
+        data = {
+            'repos': [],
+            'codename': 'trusty'
+        }
+        driver = NailgunBuildImage()
+        driver.data = data
+        mock_ub_instance = mock_ub.return_value
+        mock_ub_instance.packages = NailgunBuildImage.DEFAULT_TRUSTY_PACKAGES
+        driver.parse_operating_system()
+        mock_ub.assert_called_once_with(
+            repos=[], packages=NailgunBuildImage.DEFAULT_TRUSTY_PACKAGES)
+        self.assertEqual(driver.operating_system.packages,
+                         NailgunBuildImage.DEFAULT_TRUSTY_PACKAGES)
+
+    @mock.patch('fuel_agent.objects.DEBRepo')
+    @mock.patch('fuel_agent.objects.Ubuntu')
+    @mock.patch.object(NailgunBuildImage, '__init__')
+    def test_parse_operating_system_repos(self, mock_init, mock_ub, mock_deb):
+        mock_init.return_value = None
+        data = {
+            'repos': REPOS_SAMPLE,
+            'codename': 'trusty'
+        }
+        driver = NailgunBuildImage()
+        driver.data = data
+
+        mock_deb_expected_calls = []
+        repos = []
+        for r in REPOS_SAMPLE:
+            kwargs = {
+                'name': r['name'],
+                'uri': r['uri'],
+                'suite': r['suite'],
+                'section': r['section'],
+                'priority': r['priority']
+            }
+            mock_deb_expected_calls.append(mock.call(**kwargs))
+            repos.append(objects.DEBRepo(**kwargs))
+        driver.parse_operating_system()
+        mock_ub_instance = mock_ub.return_value
+        mock_ub_instance.repos = repos
+        mock_ub.assert_called_once_with(
+            repos=repos, packages=NailgunBuildImage.DEFAULT_TRUSTY_PACKAGES)
+        self.assertEqual(mock_deb_expected_calls,
+                         mock_deb.call_args_list[:len(REPOS_SAMPLE)])
+        self.assertEqual(driver.operating_system.repos, repos)
+
+    @mock.patch('fuel_agent.drivers.nailgun.objects.Loop')
+    @mock.patch('fuel_agent.objects.Image')
+    @mock.patch('fuel_agent.objects.Fs')
+    @mock.patch('fuel_agent.objects.PartitionScheme')
+    @mock.patch('fuel_agent.objects.ImageScheme')
+    @mock.patch.object(NailgunBuildImage, '__init__')
+    def test_parse_schemes(
+            self, mock_init, mock_imgsch, mock_partsch,
+            mock_fs, mock_img, mock_loop):
+        mock_init.return_value = None
+        data = {
+            'image_data': IMAGE_DATA_SAMPLE,
+            'output': '/some/local/path',
+        }
+        driver = NailgunBuildImage()
+        driver.data = data
+        driver.parse_schemes()
+
+        mock_fs_expected_calls = []
+        mock_img_expected_calls = []
+        images = []
+        fss = []
+        data_length = len(data['image_data'].keys())
+        for mount, image in six.iteritems(data['image_data']):
+            filename = os.path.basename(urlsplit(image['uri']).path)
+            img_kwargs = {
+                'uri': 'file://' + os.path.join(data['output'], filename),
+                'format': image['format'],
+                'container': image['container'],
+                'target_device': None
+            }
+            mock_img_expected_calls.append(mock.call(**img_kwargs))
+            images.append(objects.Image(**img_kwargs))
+
+            fs_kwargs = {
+                'device': None,
+                'mount': mount,
+                'fs_type': image['format']
+            }
+            mock_fs_expected_calls.append(mock.call(**fs_kwargs))
+            fss.append(objects.Fs(**fs_kwargs))
+
+            if mount == '/':
+                metadata_filename = filename.split('.', 1)[0] + '.yaml'
+
+        mock_imgsch_instance = mock_imgsch.return_value
+        mock_imgsch_instance.images = images
+        mock_partsch_instance = mock_partsch.return_value
+        mock_partsch_instance.fss = fss
+
+        self.assertEqual(
+            driver.metadata_uri, 'file://' + os.path.join(
+                data['output'], metadata_filename))
+        self.assertEqual(mock_img_expected_calls,
+                         mock_img.call_args_list[:data_length])
+        self.assertEqual(mock_fs_expected_calls,
+                         mock_fs.call_args_list[:data_length])
+        self.assertEqual(driver.image_scheme.images, images)
+        self.assertEqual(driver.partition_scheme.fss, fss)

fuel_agent/fuel_agent/tests/test_utils.py

@@ -34,7 +34,7 @@ class ExecuteTestCase(testtools.TestCase):
     def setUp(self):
         super(ExecuteTestCase, self).setUp()
         fake_driver = stevedore.extension.Extension('fake_driver', None, None,
-                                                    'fake_obj')
+                                                    mock.MagicMock)
         self.drv_manager = stevedore.driver.DriverManager.make_test_instance(
             fake_driver)
 
@@ -64,7 +64,8 @@ class ExecuteTestCase(testtools.TestCase):
     @mock.patch('stevedore.driver.DriverManager')
     def test_get_driver(self, mock_drv_manager):
         mock_drv_manager.return_value = self.drv_manager
-        self.assertEqual('fake_obj', utils.get_driver('fake_driver'))
+        self.assertEqual(mock.MagicMock.__name__,
+                         utils.get_driver('fake_driver').__name__)
 
     @mock.patch('jinja2.Environment')
     @mock.patch('jinja2.FileSystemLoader')
@@ -136,3 +137,26 @@ class ExecuteTestCase(testtools.TestCase):
         mock_req.side_effect = requests.exceptions.ConnectionError()
         self.assertRaises(errors.HttpUrlConnectionError,
                           utils.init_http_request, 'fake_url')
+
+    @mock.patch('fuel_agent.utils.utils.os.makedirs')
+    @mock.patch('fuel_agent.utils.utils.os.path.isdir', return_value=False)
+    def test_makedirs_if_not_exists(self, mock_isdir, mock_makedirs):
+        utils.makedirs_if_not_exists('/fake/path')
+        mock_isdir.assert_called_once_with('/fake/path')
+        mock_makedirs.assert_called_once_with('/fake/path', mode=0o755)
+
+    @mock.patch('fuel_agent.utils.utils.os.makedirs')
+    @mock.patch('fuel_agent.utils.utils.os.path.isdir', return_value=False)
+    def test_makedirs_if_not_exists_mode_given(
+            self, mock_isdir, mock_makedirs):
+        utils.makedirs_if_not_exists('/fake/path', mode=0o000)
+        mock_isdir.assert_called_once_with('/fake/path')
+        mock_makedirs.assert_called_once_with('/fake/path', mode=0o000)
+
+    @mock.patch('fuel_agent.utils.utils.os.makedirs')
+    @mock.patch('fuel_agent.utils.utils.os.path.isdir', return_value=True)
+    def test_makedirs_if_not_exists_already_exists(
+            self, mock_isdir, mock_makedirs):
+        utils.makedirs_if_not_exists('/fake/path')
+        mock_isdir.assert_called_once_with('/fake/path')
+        self.assertEqual(mock_makedirs.mock_calls, [])

fuel_agent/fuel_agent/utils/utils.py

@@ -135,8 +135,11 @@ def B2MiB(b, ceil=True):
 
 
 def get_driver(name):
-    return stevedore.driver.DriverManager(
+    LOG.debug('Trying to get driver: fuel_agent.drivers.%s', name)
+    driver = stevedore.driver.DriverManager(
         namespace='fuel_agent.drivers', name=name).driver
+    LOG.debug('Found driver: %s', driver.__name__)
+    return driver
 
 
 def render_and_save(tmpl_dir, tmpl_names, tmpl_data, file_name):
@@ -200,3 +203,12 @@ def init_http_request(url, byte_range=0):
                 'Exceeded maximum http request retries for %s' % url)
     response_obj.raise_for_status()
     return response_obj
+
+
+def makedirs_if_not_exists(path, mode=0o755):
+    """Create directory if it does not exist
+    :param path: Directory path
+    :param mode: Directory mode (Default: 0o755)
+    """
+    if not os.path.isdir(path):
+        os.makedirs(path, mode=mode)

fuel_agent/requirements.txt

@@ -3,7 +3,7 @@ eventlet>=0.13.0
 iso8601>=0.1.9
 jsonschema>=2.3.0
 oslo.config>=1.2.0
-oslo.serialization>=1.0.0
+oslo.serialization>=1.4.0
 six>=1.5.2
 pbr>=0.7.0
 Jinja2

fuel_agent/setup.cfg

@@ -14,14 +14,17 @@ packages =
 
 [entry_points]
 console_scripts =
+    # TODO(kozhukalov): rename entry point
     provision = fuel_agent.cmd.agent:provision
-    partition = fuel_agent.cmd.agent:partition
-    configdrive = fuel_agent.cmd.agent:configdrive
-    copyimage = fuel_agent.cmd.agent:copyimage
-    bootloader = fuel_agent.cmd.agent:bootloader
+    fa_partition = fuel_agent.cmd.agent:partition
+    fa_configdrive = fuel_agent.cmd.agent:configdrive
+    fa_copyimage = fuel_agent.cmd.agent:copyimage
+    fa_bootloader = fuel_agent.cmd.agent:bootloader
+    fa_build_image = fuel_agent.cmd.agent:build_image
 
 fuel_agent.drivers =
     nailgun = fuel_agent.drivers.nailgun:Nailgun
+    nailgun_build_image = fuel_agent.drivers.nailgun:NailgunBuildImage
 
 [pbr]
 autodoc_index_modules = True

nailgun/nailgun/orchestrator/provisioning_serializers.py

@@ -274,7 +274,8 @@ class ProvisioningSerializer61(ProvisioningSerializer):
                 tasks_templates.make_provisioning_images_task(
                     [consts.MASTER_ROLE],
                     attrs['repo_setup']['repos'],
-                    attrs['provision']))
+                    attrs['provision'],
+                    cluster.id))
 
         # NOTE(kozhukalov): This pre-provision task is going to be
         # removed by 7.0 because we need this only for classic way of

nailgun/nailgun/orchestrator/tasks_templates.py

@@ -162,7 +162,7 @@ def make_reboot_task(uids, task):
             'timeout': task['parameters']['timeout']}}
 
 
-def make_provisioning_images_task(uids, repos, provision_data):
+def make_provisioning_images_task(uids, repos, provision_data, cid):
     conf = {
         'repos': repos,
         'image_data': provision_data['image_data'],
@@ -175,7 +175,10 @@ def make_provisioning_images_task(uids, repos, provision_data):
 
     return make_shell_task(uids, {
         'parameters': {
-            'cmd': "fuel-image '{0}'".format(conf),
+            'cmd': ("fa_build_image "
+                    "--log-file /var/log/fuel-agent-env-{0}.log "
+                    "--data_driver nailgun_build_image "
+                    "--input_data '{1}'").format(cid, conf),
             'timeout': settings.PROVISIONING_IMAGES_BUILD_TIMEOUT,
             'retries': 1}})
 

nailgun/nailgun/test/integration/test_provisioning_serializer.py

@@ -168,7 +168,7 @@ class TestProvisioningSerializer61(BaseIntegrationTest):
             lambda task: all([
                 task['uids'] == ['master'],
                 task['type'] == 'shell',
-                task['parameters']['cmd'].startswith('fuel-image')
+                task['parameters']['cmd'].startswith('fa_build_image')
             ]),
             serialized_info['pre_provision']))
         self.assertFalse(filter(

nailgun/nailgun/test/unit/test_tasks_templates.py

@@ -156,7 +156,8 @@ class TestMakeTask(base.BaseTestCase):
                         'format': 'ext4',
                         'uri': 'http://uri'
                     }
-                }})
+                }},
+            cid=123)
 
         fuel_image_conf = {
             "image_data": {
@@ -188,7 +189,9 @@ class TestMakeTask(base.BaseTestCase):
                 'cwd': '/',
             }
         )
-        cmd = result["parameters"]["cmd"].lstrip("fuel-image '").rstrip("'")
+        cmd = result["parameters"]["cmd"].lstrip(
+            "fa_build_image --log-file /var/log/fuel-agent-env-123.log "
+            "--data_driver nailgun_build_image --input_data '").rstrip("'")
         self.assertEqual(jsonutils.loads(cmd), fuel_image_conf)
 
     def test_make_download_debian_installer_task(self):