# Copyright 2015 - Rackspace Hosting # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Base LP handler for building apps""" import errno import io import json import logging import os import random import string import time import docker from docker import errors from oslo_config import cfg from requests.packages.urllib3 import exceptions as req_exp from solum.common import exception as exc from solum.common import solum_swiftclient from solum.openstack.common import log as solum_log from solum.uploaders import tenant_logger from solum.worker.app_handlers import utils from swiftclient import exceptions as swiftexp LOG = solum_log.getLogger(__name__) cfg.CONF.import_opt('task_log_dir', 'solum.worker.config', group='worker') cfg.CONF.import_opt('docker_daemon_url', 'solum.worker.config', group='worker') cfg.CONF.import_opt('docker_build_timeout', 'solum.worker.config', group='worker') cfg.CONF.import_opt('container_mem_limit', 'solum.worker.config', group='worker') log_dir = cfg.CONF.worker.task_log_dir docker_daemon_url = cfg.CONF.worker.docker_daemon_url build_timeout = cfg.CONF.worker.docker_build_timeout mem_limit = cfg.CONF.worker.container_mem_limit MAX_GIT_CLONE_RETRY = 5 GIT_CLONE_TIMEOUT = 900 # 15 minutes cloner_gid = os.getgid() class BaseHandler(object): def __init__(self, context, assembly, image_storage): self.context = context self.assembly = assembly self.image_storage = image_storage self._docker = None self.docker_cmd_uid = cfg.CONF.run_container_cmd_as self.cloner_image = None self.images = list() self.containers = list() self.work_dir = None def __enter__(self): return self def __exit__(self, exc_type, exc_value, exc_traceback): self.close() @property def docker(self): if self._docker is None: self._docker = docker.Client(base_url=docker_daemon_url) return self._docker def _get_tenant_logger(self, stage): return tenant_logger.TenantLogger(self.context, self.assembly, log_dir, stage) def close(self): for ct in self.containers: if ct: try: self.docker.remove_container(container=ct.get('Id')) except (errors.DockerException, errors.APIError) as e: LOG.warning('Failed to remove container %s, %s' % (ct.get('Id'), str(e))) for img in self.images: try: self.docker.remove_image(image=img, force=True) except (errors.DockerException, errors.APIError) as e: LOG.warning('Failed to remove docker image %s, %s' % (img, str(e))) if self.work_dir: self._remove_cloned_repo(self.work_dir) try: utils.rm_tree(self.work_dir) except OSError as e: if e.errno != errno.ENOENT: LOG.critical('critical: cannot remove dir %s,' ' disk may be full.' % self.work_dir) if self.cloner_image: try: self.docker.remove_image(image=self.cloner_image, force=True) except (errors.DockerException, errors.APIError) as e: LOG.error('Error in removing docker image %s, %s' % (self.cloner_image, str(e))) def _validate_pub_repo(self, repo_url): pass @utils.retry def _remove_cloned_repo(self, destination): if not os.path.exists(destination): return 0 result = 1 try: ct = self.docker.create_container( image=self.cloner_image, user=str(self.docker_cmd_uid), command=['rm', '-rf', '/tmp/code']) self.docker.start(container=ct.get('Id'), binds={destination: '/tmp'}) result = self.docker.wait(container=ct.get('Id')) self.docker.remove_container(container=ct.get('Id')) except (errors.DockerException, errors.APIError) as e: clone_dir = '{}/code'.format(destination) LOG.error('Error in remove cloned repo %s, %s' % (clone_dir, str(e))) return result def _clone_repo(self, repo_url, destination, logger, revision='master'): # Clone a repo with the constraints of disk and memory usage # Need to consider limiting network bandwidth as well. container_dest = '/tmp/code' if utils.is_git_sha(revision): clone_cmd = ('git clone {url} {dst} &&' ' cd {dst} &&' ' git checkout -B solum {rev} &&' ' echo sha=$(git log -1 --pretty=%H)').format( url=repo_url, dst=container_dest, rev=revision) else: clone_cmd = ('git clone -b {branch} --depth 1 {url} {dst} &&' ' cd {dst} &&' ' echo sha=$(git log -1 --pretty=%H)').format( branch=revision, url=repo_url, dst=container_dest) timeout_clone = 'timeout --signal=SIGKILL {t} {clone}'.format( t=GIT_CLONE_TIMEOUT, clone=clone_cmd) dockerfile = ('FROM solum/cloner\n' 'RUN groupadd -f -g {gid} s-cloner-group\n' 'RUN useradd -s /bin/bash -u {uid} -g {gid} s-cloner\n' 'USER s-cloner\n' 'CMD {cmd}').format(uid=self.docker_cmd_uid, gid=cloner_gid, cmd=timeout_clone) ranid = ''.join(random.choice(string.digits) for _ in range(5)) self.cloner_image = '{}-cloner-{}'.format(self.docker_cmd_uid, ranid) try: self._docker_build_with_retry( self.cloner_image, logger, pull=False, fileobj=io.BytesIO(dockerfile.encode('utf-8'))) ct = self.docker.create_container(image=self.cloner_image, mem_limit=mem_limit, memswap_limit=-1) except (errors.DockerException, errors.APIError) as e: logger.log(logging.ERROR, 'Pre git clone stage failed.') LOG.error('Error in building/creating container for cloning,' ' assembly: %s, %s' % (self.assembly.uuid, str(e))) return head_sha = None for i in range(MAX_GIT_CLONE_RETRY): # retry cloning try: self.docker.start(container=ct.get('Id'), binds={destination: '/tmp'}) for line in self.docker.attach(container=ct.get('Id'), stream=True): if line.startswith('sha='): head_sha = line.replace('sha=', '').strip() else: logger.log(logging.INFO, line) except (errors.DockerException, errors.APIError) as e: logger.log(logging.ERROR, 'Got an error in cloning the repo,' ' max retry %s times. Repo: %s,' ' revision: %s' % (MAX_GIT_CLONE_RETRY, repo_url, revision)) LOG.error('Error in cloning. assembly: %s, repo: %s,' ' rev: %s, %s' % (self.assembly.uuid, repo_url, revision, str(e))) if head_sha: logger.log(logging.INFO, 'Finished cloning repo.') break elif i < MAX_GIT_CLONE_RETRY - 1: clone_dir = '{}/code'.format(destination) res = self._remove_cloned_repo(destination) if res != 0: LOG.critical('critical: cannot remove dir %s,' ' disk may be full.' % clone_dir) time.sleep(3) try: self.docker.remove_container(container=ct.get('Id')) except (errors.DockerException, errors.APIError): pass return head_sha def _gen_docker_ignore(self, path, prefix=None): # Exclude .git from the docker build context content = '{}/.git'.format(prefix) if prefix else '.git' try: with open('{}/.dockerignore'.format(path), 'w') as f: f.write(content) except OSError: pass def _docker_build(self, tag, logger, timeout, limits, path=None, dockerfile=None, fileobj=None, forcerm=True, quiet=True, nocache=False, pull=True): success = 1 try: for l in self.docker.build(path=path, dockerfile=dockerfile, fileobj=fileobj, tag=tag, timeout=timeout, forcerm=forcerm, quiet=quiet, nocache=nocache, pull=pull, container_limits=limits): try: info = json.loads(l).get('stream', '') if info: if 'successfully built' in info.lower(): success = 0 else: err = json.loads(l).get('errorDetail', '') if err: logger.log(logging.ERROR, err) except ValueError: pass except req_exp.ReadTimeoutError: logger.log(logging.ERROR, 'docker build timed out, max value: %s' % timeout) except (errors.DockerException, errors.APIError) as e: LOG.error('Error in building docker image %s, assembly: %s, %s' % (tag, self.assembly.uuid, str(e))) return success def _docker_build_with_retry(self, tag, logger, path=None, dockerfile=None, fileobj=None, forcerm=True, quiet=True, limits=None, pull=True, timeout=build_timeout): limits = limits or {'memory': mem_limit, 'memswap': -1} result = self._docker_build(tag, logger, timeout, limits, path=path, dockerfile=dockerfile, fileobj=fileobj, forcerm=forcerm, quiet=quiet, pull=pull, nocache=False) if result == 0: return 0 time.sleep(2) result = self._docker_build(tag, logger, timeout, limits, path=path, dockerfile=dockerfile, fileobj=fileobj, forcerm=forcerm, quiet=quiet, pull=pull, nocache=True) return result @utils.retry def _docker_save(self, image, output): result = 1 try: lp = self.docker.get_image(image) with open(output, 'w') as f: f.write(lp.data) result = 0 except (OSError, errors.DockerException, errors.APIError) as e: LOG.error('Error saving docker image, %s' % str(e)) return result @utils.retry def _docker_load(self, path): result = 1 try: with open(path, 'rb') as f: self.docker.load_image(f) result = 0 except (OSError, errors.DockerException, errors.APIError) as e: LOG.error('Error in loading docker image, %s' % str(e)) return result def _persist_to_backend(self, local_file, swift_container, swift_obj, logger): loc = None if (self.image_storage == 'glance' or self.image_storage == 'docker_registry'): return loc elif self.image_storage == 'swift': swift = solum_swiftclient.SwiftClient(self.context) try: swift.upload(local_file, swift_container, swift_obj) loc = swift_obj except exc.InvalidObjectSizeError: logger.log(logging.INFO, 'Image with size exceeding 5GB' ' is not supported') except swiftexp.ClientException as e: LOG.error('Error in persisting artifact to swift, %s' % str(e)) return loc def unittest_app(self, *args): """Interface to implement in derived class.""" pass def build_app(self, *args): """Interface to implement in derived class.""" pass