406 lines
14 KiB
Python
406 lines
14 KiB
Python
from concurrent import futures
|
|
import contextlib
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
import shutil
|
|
import sys
|
|
import tempfile
|
|
|
|
import docker
|
|
import git
|
|
|
|
from fuel_ccp.common import jinja_utils
|
|
from fuel_ccp import config
|
|
|
|
BUILD_TIMEOUT = 2 ** 16 # in seconds
|
|
|
|
CONF = config.CONF
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
_SHUTDOWN = False
|
|
|
|
|
|
def create_rendered_dockerfile(path, name, tmp_path, config):
|
|
def copy_sources(project_name, cont_dir):
|
|
tmp_dir = os.path.join(tmp_path, name, project_name)
|
|
|
|
git_url = config['sources'].get(project_name, {}).get('git_url')
|
|
source_dir = config['sources'].get(project_name, {}).get('source_dir')
|
|
|
|
if not git_url and not source_dir:
|
|
LOG.error('%s: Neither git_url nor source_dir specified for '
|
|
'project %s (exactly one should be specified)',
|
|
name, project_name)
|
|
sys.exit(1)
|
|
|
|
if git_url and source_dir:
|
|
LOG.error('%s: Both git_url and source_dir specified for '
|
|
'project %s (exactly one should be specified)',
|
|
name, project_name)
|
|
sys.exit(1)
|
|
|
|
if git_url:
|
|
LOG.info('%s: Cloning repository "%s"', name, git_url)
|
|
repo = git.Repo.clone_from(git_url, tmp_dir)
|
|
ref = config['sources'][project_name]['git_ref']
|
|
LOG.info('%s: Changing reference to "%s"', name, ref)
|
|
repo.git.checkout(ref)
|
|
LOG.info('%s: Repository %s has been cloned', name, git_url)
|
|
|
|
if source_dir:
|
|
LOG.info('%s: Using local directory %s', name, source_dir)
|
|
shutil.copytree(source_dir, tmp_dir)
|
|
|
|
return 'COPY %s %s' % (project_name, cont_dir)
|
|
|
|
LOG.info('%s: Rendering dockerfile', name)
|
|
src_dir = os.path.dirname(path)
|
|
dest_dir = os.path.join(tmp_path, name)
|
|
os.makedirs(dest_dir)
|
|
dockerfilename = os.path.join(dest_dir, 'Dockerfile')
|
|
content = jinja_utils.jinja_render(
|
|
path, config['render'], [copy_sources])
|
|
with open(dockerfilename, 'w') as f:
|
|
f.write(content)
|
|
|
|
for filename in os.listdir(src_dir):
|
|
if 'Dockerfile' in filename:
|
|
continue
|
|
full_filename = os.path.join(src_dir, filename)
|
|
if os.path.isfile(full_filename):
|
|
shutil.copy(full_filename, dest_dir)
|
|
elif os.path.isdir(full_filename):
|
|
shutil.copytree(full_filename, os.path.join(dest_dir, filename))
|
|
|
|
return dockerfilename
|
|
|
|
|
|
def find_dockerfiles(repository_name, match=True):
|
|
dockerfiles = {}
|
|
repository_dir = os.path.join(CONF.repositories.path, repository_name)
|
|
|
|
namespace = CONF.images.namespace
|
|
if CONF.builder.push and CONF.registry.address:
|
|
namespace = '%s/%s' % (CONF.registry.address, namespace)
|
|
|
|
for root, __, files in os.walk(repository_dir):
|
|
if 'Dockerfile.j2' in files:
|
|
path = os.path.join(root, 'Dockerfile.j2')
|
|
else:
|
|
continue
|
|
name = os.path.basename(os.path.dirname(path))
|
|
dockerfiles[name] = {
|
|
'name': name,
|
|
'full_name': '%s/%s:%s' % (namespace, name, CONF.images.tag),
|
|
'path': path,
|
|
'parent': None,
|
|
'children': [],
|
|
'match': match,
|
|
'build_result': None,
|
|
'push_result': None
|
|
}
|
|
|
|
if len(dockerfiles) == 0:
|
|
msg = 'No dockerfile for %s found'
|
|
if CONF.repositories.skip_empty:
|
|
LOG.debug(msg, repository_name)
|
|
else:
|
|
LOG.error(msg, repository_name)
|
|
sys.exit(1)
|
|
|
|
return dockerfiles
|
|
|
|
|
|
IMAGE_FULL_NAME_RE = r"((?P<namespace>[\w:\.-]+)/){0,2}" \
|
|
"(?P<name>[\w_-]+)" \
|
|
"(:(?P<tag>[\w_\.-]+))?"
|
|
IMAGE_FULL_NAME_PATTERN = re.compile(IMAGE_FULL_NAME_RE)
|
|
# This regex is needed for matching not yet rendered images
|
|
NOT_RENDERED_IMAGE_PATTERN = (r"((?P<namespace>[\w:\.\-}{ ]+)/){0,2}"
|
|
r"(?P<name>[\w_\-}{ ]+)"
|
|
r"(:(?P<tag>[\w_\.\-}{ ]+))?")
|
|
|
|
DOCKER_FILE_FROM_PATTERN = re.compile(
|
|
r"^\s?FROM\s+{}\s?$".format(NOT_RENDERED_IMAGE_PATTERN), re.MULTILINE
|
|
)
|
|
|
|
|
|
def find_dependencies(dockerfiles):
|
|
for name, dockerfile in dockerfiles.items():
|
|
with open(dockerfile['path']) as f:
|
|
content = f.read()
|
|
|
|
matcher = DOCKER_FILE_FROM_PATTERN.search(content)
|
|
if not matcher:
|
|
raise RuntimeError(
|
|
"FROM clause was not found in dockerfile for image: " + name
|
|
)
|
|
|
|
parent_ns = matcher.group("namespace")
|
|
if not parent_ns:
|
|
continue
|
|
parent_name = matcher.group("name")
|
|
|
|
dockerfile['parent'] = dockerfiles[parent_name]
|
|
dockerfiles[parent_name]['children'].append(dockerfile)
|
|
|
|
|
|
def build_dockerfile(dc, dockerfile):
|
|
LOG.info("%s: Starting image build", dockerfile['name'])
|
|
for line in dc.build(rm=True,
|
|
forcerm=True,
|
|
nocache=CONF.builder.no_cache,
|
|
tag=dockerfile['full_name'],
|
|
path=os.path.dirname(dockerfile['path'])):
|
|
if _SHUTDOWN:
|
|
raise RuntimeError("Building '{}' was interrupted".format(
|
|
dockerfile['name']
|
|
))
|
|
build_data = json.loads(line.decode("UTF-8"))
|
|
if 'stream' in build_data:
|
|
LOG.debug('%s: %s' % (dockerfile['name'],
|
|
build_data['stream'].rstrip()))
|
|
if 'errorDetail' in build_data:
|
|
LOG.error('%s: %s' % (dockerfile['name'],
|
|
build_data['errorDetail']['message']))
|
|
dockerfile['build_result'] = 'Failure'
|
|
return
|
|
dockerfile['build_result'] = 'Success'
|
|
LOG.info("%s: Build succeeded", dockerfile['name'])
|
|
|
|
|
|
def push_dockerfile(dc, dockerfile):
|
|
if dockerfile['build_result'] == 'Failure':
|
|
dockerfile['push_result'] = 'Failure'
|
|
LOG.error("%s: Push will be skipped due to build failure",
|
|
dockerfile['name'])
|
|
return
|
|
if CONF.registry.username and CONF.registry.password:
|
|
dc.login(username=CONF.registry.username,
|
|
password=CONF.registry.password,
|
|
registry=CONF.registry.address)
|
|
for line in dc.push(dockerfile['full_name'],
|
|
stream=True,
|
|
insecure_registry=CONF.registry.insecure):
|
|
build_data = json.loads(line.decode("UTF-8"))
|
|
|
|
status = build_data.get('status', '')
|
|
|
|
if status:
|
|
LOG.debug('%s: %s' % (dockerfile['name'], status))
|
|
if build_data.get('progress'):
|
|
LOG.debug('%s: %s' % (
|
|
dockerfile['name'], build_data['progress'].rstrip()))
|
|
|
|
if ('Layer already exists' in status and
|
|
not dockerfile['push_result']):
|
|
dockerfile['push_result'] = 'Exists'
|
|
elif 'errorDetail' in build_data:
|
|
LOG.error('%s: %s', dockerfile['name'],
|
|
build_data['errorDetail']['message'])
|
|
dockerfile['push_result'] = 'Failure'
|
|
elif status == 'Pushed' or 'Mounted from' in status:
|
|
dockerfile['push_result'] = 'Success'
|
|
if dockerfile['push_result'] == 'Success':
|
|
LOG.info("%s: Push into %s registry finished", dockerfile['name'],
|
|
CONF.registry.address)
|
|
elif dockerfile['push_result'] == 'Exists':
|
|
LOG.info("%s: Already in %s registry", dockerfile['name'],
|
|
CONF.registry.address)
|
|
|
|
|
|
def process_dockerfile(dockerfile, tmp_dir, config, executor, future_list,
|
|
ready_images):
|
|
path = create_rendered_dockerfile(
|
|
dockerfile['path'], dockerfile['name'], tmp_dir, config)
|
|
dockerfile['path'] = path
|
|
with contextlib.closing(docker.Client(
|
|
timeout=CONF.registry.timeout)) as dc:
|
|
build_dockerfile(dc, dockerfile)
|
|
if CONF.builder.push and CONF.registry.address:
|
|
push_dockerfile(dc, dockerfile)
|
|
|
|
for child in dockerfile['children']:
|
|
if child['match'] or (CONF.builder.keep_image_tree_consistency and
|
|
child['name'] in ready_images):
|
|
if dockerfile['build_result'] == 'Failure':
|
|
LOG.error("%s: Build will be skipped due to parent image (%s) "
|
|
"build failure", child['name'], dockerfile['name'])
|
|
child['build_result'] = 'Failure'
|
|
if CONF.builder.push:
|
|
child['push_result'] = 'Failure'
|
|
else:
|
|
submit_dockerfile_processing(child, tmp_dir, config, executor,
|
|
future_list, ready_images)
|
|
|
|
|
|
def submit_dockerfile_processing(dockerfile, tmp_dir, config, executor,
|
|
future_list, ready_images):
|
|
future_list.append(executor.submit(
|
|
process_dockerfile, dockerfile, tmp_dir, config,
|
|
executor, future_list, ready_images
|
|
))
|
|
|
|
|
|
def match_not_ready_base_dockerfiles(dockerfile, ready_images):
|
|
while True:
|
|
parent = dockerfile['parent']
|
|
if parent is None or parent['match'] or parent['name'] in ready_images:
|
|
break
|
|
parent['match'] = True
|
|
dockerfile = parent
|
|
|
|
|
|
def get_ready_image_names():
|
|
with contextlib.closing(docker.Client(
|
|
timeout=CONF.registry.timeout)) as dc:
|
|
ready_images = []
|
|
for image in dc.images():
|
|
if image["RepoTags"]:
|
|
for repo_tag in image["RepoTags"]:
|
|
matcher = IMAGE_FULL_NAME_PATTERN.match(repo_tag)
|
|
if not matcher:
|
|
continue
|
|
ns = matcher.group("namespace")
|
|
name = matcher.group("name")
|
|
tag = matcher.group("tag")
|
|
if CONF.images.namespace == ns and CONF.images.tag == tag:
|
|
ready_images.append(name)
|
|
return ready_images
|
|
|
|
|
|
def match_dockerfiles_by_component(dockerfiles, component, ready_images):
|
|
pattern = re.compile(re.escape(component))
|
|
matched_dockerfiles = list(filter(pattern.match, dockerfiles.keys()))
|
|
if matched_dockerfiles:
|
|
LOG.info("Component \"%s\" matches: %s", component,
|
|
", ".join(matched_dockerfiles))
|
|
else:
|
|
raise RuntimeError("Component \"%s\" doesn't match any "
|
|
"dockerfile" % component)
|
|
for dockerfile in matched_dockerfiles:
|
|
dockerfiles[dockerfile]['match'] = True
|
|
if CONF.builder.build_base_images_if_not_exist:
|
|
match_not_ready_base_dockerfiles(
|
|
dockerfiles[dockerfile], ready_images)
|
|
|
|
|
|
def wait_futures(future_list, skip_errors=False):
|
|
while future_list:
|
|
future = future_list[0]
|
|
if future.done():
|
|
future_list.pop(0)
|
|
continue
|
|
try:
|
|
# we need to use timeout because in this case python
|
|
# thread wakes up time to time to check timeout and don't
|
|
# block signal processing
|
|
future.result(timeout=BUILD_TIMEOUT)
|
|
except Exception as ex:
|
|
if skip_errors:
|
|
LOG.error(str(ex))
|
|
else:
|
|
raise
|
|
|
|
|
|
def _get_config():
|
|
cfg = {'render': dict(CONF.images._items())}
|
|
if CONF.registry.address:
|
|
cfg['render']['namespace'] = '%s/%s' % (
|
|
CONF.registry.address, cfg['render']['namespace'])
|
|
|
|
cfg['render'].update(CONF.versions._items())
|
|
cfg['sources'] = CONF.sources
|
|
|
|
return cfg
|
|
|
|
|
|
def _get_summary(dockerfiles):
|
|
LOG.info('#' * 50)
|
|
LOG.info('Summary:')
|
|
|
|
build_succeeded = [d['name'] for d in dockerfiles.values()
|
|
if d['build_result'] == 'Success']
|
|
if build_succeeded:
|
|
LOG.info('%d image(s) build succeeded: %s' % (
|
|
len(build_succeeded), ', '.join(build_succeeded)))
|
|
|
|
build_failed = [d['name'] for d in dockerfiles.values()
|
|
if d['build_result'] == 'Failure']
|
|
if build_failed:
|
|
LOG.error('%d image(s) build failed: %s' % (
|
|
len(build_failed), ', '.join(build_failed)))
|
|
|
|
push_succeeded = [d['name'] for d in dockerfiles.values()
|
|
if d['push_result'] == 'Success']
|
|
if push_succeeded:
|
|
LOG.info('%d image(s) push succeeded: %s' % (
|
|
len(push_succeeded), ', '.join(push_succeeded)))
|
|
|
|
already_pushed = [d['name'] for d in dockerfiles.values()
|
|
if d['push_result'] == 'Exists']
|
|
if already_pushed:
|
|
LOG.info('%d image(s) already in registry: %s' % (
|
|
len(already_pushed), ', '.join(already_pushed)))
|
|
|
|
push_failed = [d['name'] for d in dockerfiles.values()
|
|
if d['push_result'] == 'Failure']
|
|
if push_failed:
|
|
LOG.error('%d image(s) push failed: %s' % (
|
|
len(push_failed), ', '.join(push_failed)))
|
|
LOG.info('#' * 50)
|
|
|
|
if build_failed or push_failed:
|
|
return False
|
|
return True
|
|
|
|
|
|
def build_components(components=None):
|
|
tmp_dir = tempfile.mkdtemp()
|
|
|
|
config = _get_config()
|
|
dockerfiles = {}
|
|
try:
|
|
match = not bool(components)
|
|
for repository_def in CONF.repositories.repos:
|
|
dockerfiles.update(
|
|
find_dockerfiles(repository_def['name'], match=match))
|
|
|
|
find_dependencies(dockerfiles)
|
|
|
|
ready_images = get_ready_image_names()
|
|
|
|
if components is not None:
|
|
for component in components:
|
|
match_dockerfiles_by_component(dockerfiles, component,
|
|
ready_images)
|
|
|
|
with futures.ThreadPoolExecutor(max_workers=CONF.builder.workers) as (
|
|
executor):
|
|
future_list = []
|
|
try:
|
|
for dockerfile in dockerfiles.values():
|
|
if dockerfile['match'] and (
|
|
dockerfile['parent'] is None or
|
|
not dockerfile['parent']['match']):
|
|
submit_dockerfile_processing(
|
|
dockerfile, tmp_dir, config, executor,
|
|
future_list, ready_images)
|
|
|
|
wait_futures(future_list)
|
|
except SystemExit:
|
|
global _SHUTDOWN
|
|
_SHUTDOWN = True
|
|
for future in future_list:
|
|
future.cancel()
|
|
wait_futures(future_list, skip_errors=True)
|
|
raise
|
|
finally:
|
|
build_succeeded = _get_summary(dockerfiles)
|
|
shutil.rmtree(tmp_dir)
|
|
if not build_succeeded:
|
|
sys.exit(1)
|