From 37842bcc973fd00845216b0496cbb02e207c0c10 Mon Sep 17 00:00:00 2001 From: Craig Tracey Date: Fri, 14 Nov 2014 18:44:36 -0500 Subject: [PATCH] Refactor with builder support Refactor the builders to so that a user may specify one of 2 paths: system packages or docker containers. This change also encompasses a bunch of other minor changes. This is being checked in in order to get some minimally viable changes upstream. --- .gitignore | 1 + giftwrap/builder.py | 54 +++------ giftwrap/builders/__init__.py | 0 giftwrap/builders/docker_builder.py | 169 +++++++++++++++++++++++++++ giftwrap/builders/package_builder.py | 74 ++++++++++++ giftwrap/openstack_git_repo.py | 6 +- giftwrap/openstack_project.py | 5 +- giftwrap/package.py | 9 +- giftwrap/settings.py | 9 +- giftwrap/shell.py | 5 +- giftwrap/templates/Dockerfile.jinja2 | 12 ++ giftwrap/util.py | 6 +- requirements.txt | 1 + sample.yml | 29 ++--- setup.cfg | 3 + tox.ini | 2 +- 16 files changed, 319 insertions(+), 66 deletions(-) create mode 100644 giftwrap/builders/__init__.py create mode 100644 giftwrap/builders/docker_builder.py create mode 100644 giftwrap/builders/package_builder.py create mode 100644 giftwrap/templates/Dockerfile.jinja2 diff --git a/.gitignore b/.gitignore index 14b7c58..77e237c 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ build pbr*.egg *.pyc *.sw? +*.deb diff --git a/giftwrap/builder.py b/giftwrap/builder.py index 6a64176..c690ad1 100644 --- a/giftwrap/builder.py +++ b/giftwrap/builder.py @@ -15,13 +15,6 @@ # License for the specific language governing permissions and limitations import logging -import os -import sys - -from giftwrap.gerrit import GerritReview -from giftwrap.openstack_git_repo import OpenstackGitRepo -from giftwrap.package import Package -from giftwrap.util import execute LOG = logging.getLogger(__name__) @@ -30,39 +23,26 @@ class Builder(object): def __init__(self, spec): self._spec = spec + self.settings = spec.settings + + def _build(self): + raise NotImplementedError() + + def _validate_settings(self): + raise NotImplementedError() def build(self): - """ this is where all the magic happens """ + self._validate_settings() + self._build() - try: - spec = self._spec - for project in self._spec.projects: - LOG.info("Beginning to build '%s'", project.name) - os.makedirs(project.install_path) - LOG.info("Fetching source code for '%s'", project.name) - repo = OpenstackGitRepo(project.giturl, project.gitref) - repo.clone(project.install_path) - review = GerritReview(repo.change_id, project.git_path) +from giftwrap.builders.package_builder import PackageBuilder +from giftwrap.builders.docker_builder import DockerBuilder - LOG.info("Creating the virtualenv for '%s'", project.name) - execute(project.venv_command, project.install_path) - LOG.info("Installing '%s' pip dependencies to the virtualenv", - project.name) - execute(project.install_command % - review.build_pip_dependencies(string=True), - project.install_path) - - LOG.info("Installing '%s' to the virtualenv", project.name) - execute(".venv/bin/python setup.py install", - project.install_path) - - if not spec.settings.all_in_one: - pkg = Package(project.package_name, project.version, - project.install_path, True) - pkg.build() - - except Exception as e: - LOG.exception("Oops. Something went wrong. Error was:\n%s", e) - sys.exit(-1) +def create_builder(spec): + if spec.settings.build_type == 'package': + return PackageBuilder(spec) + elif spec.settings.build_type == 'docker': + return DockerBuilder(spec) + raise Exception("Unknown build_type: '%s'", spec.settings.build_type) diff --git a/giftwrap/builders/__init__.py b/giftwrap/builders/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/giftwrap/builders/docker_builder.py b/giftwrap/builders/docker_builder.py new file mode 100644 index 0000000..0711138 --- /dev/null +++ b/giftwrap/builders/docker_builder.py @@ -0,0 +1,169 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2014, Craig Tracey +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations + +import docker +import json +import logging +import os +import re +import tempfile + +from giftwrap.builder import Builder + +LOG = logging.getLogger(__name__) + +import jinja2 + +DEFAULT_TEMPLATE_FILE = os.path.join(os.path.dirname( + os.path.dirname(__file__)), + 'templates/Dockerfile.jinja2') +APT_REQUIRED_PACKAGES = [ + 'libffi-dev', + 'libxml2-dev', + 'libxslt1-dev', + 'git', + 'wget', + 'curl', + 'libldap2-dev', + 'libsasl2-dev', + 'libssl-dev', + 'python-dev', + 'libmysqlclient-dev', + 'python-virtualenv', + 'python-pip', + 'build-essential' +] + + +class DockerBuilder(Builder): + + def __init__(self, spec): + self.template = DEFAULT_TEMPLATE_FILE + self.base_image = 'ubuntu:12.04' + self.maintainer = 'maintainer@example.com' + self.envvars = {'DEBIAN_FRONTEND': 'noninteractive'} + self._paths = [] + super(DockerBuilder, self).__init__(spec) + + def _validate_settings(self): + if not self.settings.all_in_one: + LOG.warn("The Docker builder does not support all-in-one") + + def _get_prep_commands(self): + commands = [] + commands.append('apt-get update && apt-get install -y %s' % + ' '.join(APT_REQUIRED_PACKAGES)) + return commands + + def _get_build_commands(self, src_path): + + commands = [] + commands.append('mkdir -p %s' % src_path) + + for project in self._spec.projects: + if project.system_dependencies: + commands.append('apt-get update && apt-get install -y %s' % + ' '.join(project.system_dependencies)) + + project_src_path = os.path.join(src_path, project.name) + commands.append('git clone %s -b %s %s' % (project.giturl, + project.gitref, + project_src_path)) + commands.append('mkdir -p %s' % + os.path.dirname(project.install_path)) + commands.append('virtualenv --system-site-packages %s' % + project.install_path) + + project_bin_path = os.path.join(project.install_path, 'bin') + self._paths.append(project_bin_path) + venv_python_path = os.path.join(project_bin_path, 'python') + venv_pip_path = os.path.join(project_bin_path, 'pip') + + if project.pip_dependencies: + commands.append("%s install %s" % (venv_pip_path, + ' '.join(project.pip_dependencies))) + commands.append('cd %s && %s setup.py install && cd -' % + (project_src_path, venv_python_path)) + commands.append("%s install pbr" % venv_pip_path) + + return commands + + def _get_cleanup_commands(self, src_path): + commands = [] + commands.append('rm -rf %s' % src_path) + return commands + + def _set_path(self): + path = ":".join(self._paths) + self.envvars['PATH'] = "%s:$PATH" % path + + def _render_dockerfile(self, extra_vars): + template_vars = self.__dict__.update(extra_vars) + template_loader = jinja2.FileSystemLoader(searchpath='/') + template_env = jinja2.Environment(loader=template_loader) + template = template_env.get_template(self.template) + return template.render(template_vars) + + def _build(self): + src_path = '/tmp/build' + commands = self._get_prep_commands() + commands += self._get_build_commands(src_path) + commands += self._get_cleanup_commands(src_path) + self._set_path() + dockerfile_contents = self._render_dockerfile(locals()) + + tempdir = tempfile.mkdtemp() + dockerfile = os.path.join(tempdir, 'Dockerfile') + with open(dockerfile, "w") as w: + w.write(dockerfile_contents) + + docker_client = docker.Client(base_url='unix://var/run/docker.sock', + version='1.10', timeout=10) + + build_result = docker_client.build(path=tempdir, stream=True, + tag='openstack-9.0:bbc6') + for line in build_result: + LOG.info(line.strip()) + + # I borrowed this from docker/stackbrew, should cull it down + # to be more sane. + def _parse_result(self, build_result): + build_success_re = r'^Successfully built ([a-f0-9]+)\n$' + if isinstance(build_result, tuple): + img_id, logs = build_result + return img_id, logs + else: + lines = [line for line in build_result] + try: + parsed_lines = [json.loads(e).get('stream', '') for e in lines] + except ValueError: + # sometimes all the data is sent on a single line ???? + # + # ValueError: Extra data: line 1 column 87 - line 1 column + # 33268 (char 86 - 33267) + line = lines[0] + # This ONLY works because every line is formatted as + # {"stream": STRING} + parsed_lines = [ + json.loads(obj).get('stream', '') for obj in + re.findall('{\s*"stream"\s*:\s*"[^"]*"\s*}', line) + ] + + for line in parsed_lines: + match = re.match(build_success_re, line) + if match: + return match.group(1), parsed_lines + return None, lines diff --git a/giftwrap/builders/package_builder.py b/giftwrap/builders/package_builder.py new file mode 100644 index 0000000..af4a1e2 --- /dev/null +++ b/giftwrap/builders/package_builder.py @@ -0,0 +1,74 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2014, Craig Tracey +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations + +import logging +import os +import shutil + +from giftwrap.gerrit import GerritReview +from giftwrap.openstack_git_repo import OpenstackGitRepo +from giftwrap.package import Package +from giftwrap.util import execute + +LOG = logging.getLogger(__name__) + + +from giftwrap.builder import Builder + + +class PackageBuilder(Builder): + + def __init__(self, spec): + self._all_in_one = False + super(PackageBuilder, self).__init__(spec) + + def _validate_settings(self): + pass + + def _build(self): + spec = self._spec + for project in self._spec.projects: + LOG.info("Beginning to build '%s'", project.name) + if (os.path.exists(project.install_path) and + spec.settings.force_overwrite): + LOG.info("force_overwrite is set, so removing " + "existing path '%s'" % project.install_path) + shutil.rmtree(project.install_path) + os.makedirs(project.install_path) + + LOG.info("Fetching source code for '%s'", project.name) + repo = OpenstackGitRepo(project.giturl, project.gitref) + repo.clone(project.install_path) + review = GerritReview(repo.change_id, project.git_path) + + LOG.info("Creating the virtualenv for '%s'", project.name) + execute(project.venv_command, project.install_path) + + LOG.info("Installing '%s' pip dependencies to the virtualenv", + project.name) + execute(project.install_command % + review.build_pip_dependencies(string=True), + project.install_path) + + LOG.info("Installing '%s' to the virtualenv", project.name) + execute(".venv/bin/python setup.py install", + project.install_path) + + if not spec.settings.all_in_one: + pkg = Package(project.package_name, project.version, + project.install_path, True, + spec.settings.force_overwrite) + pkg.build() diff --git a/giftwrap/openstack_git_repo.py b/giftwrap/openstack_git_repo.py index 347f544..fa2536f 100644 --- a/giftwrap/openstack_git_repo.py +++ b/giftwrap/openstack_git_repo.py @@ -65,7 +65,7 @@ class OpenstackGitRepo(object): self._committed_date = None def clone(self, outdir): - LOG.debug("Cloning '%s' to '%s'", self.url, outdir) + LOG.info("Cloning '%s' to '%s'", self.url, outdir) self._repo = Repo.clone_from(self.url, outdir) git = self._repo.git git.checkout(self.ref) @@ -83,6 +83,6 @@ class OpenstackGitRepo(object): raise Exception("Unable to find commit for date %s", datetime.datetime.fromtimestamp(date)) git = self._repo.git - LOG.debug("Reset repo '%s' to commit at '%s'", self.url, - time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(date))) + LOG.info("Reset repo '%s' to commit at '%s'", self.url, + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(date))) git.checkout(commit_date_sha) diff --git a/giftwrap/openstack_project.py b/giftwrap/openstack_project.py index 2303622..010bbf6 100644 --- a/giftwrap/openstack_project.py +++ b/giftwrap/openstack_project.py @@ -34,7 +34,8 @@ class OpenstackProject(object): def __init__(self, settings, name, version=None, gitref=None, giturl=None, venv_command=None, install_command=None, install_path=None, - package_name=None, stackforge=False): + package_name=None, stackforge=False, system_dependencies=[], + pip_dependencies=[]): self._settings = settings self.name = name self._version = version @@ -46,6 +47,8 @@ class OpenstackProject(object): self._package_name = package_name self.stackforge = stackforge self._git_path = None + self.system_dependencies = system_dependencies + self.pip_dependencies = pip_dependencies @property def version(self): diff --git a/giftwrap/package.py b/giftwrap/package.py index 8e7ffd4..3baa2c1 100644 --- a/giftwrap/package.py +++ b/giftwrap/package.py @@ -26,11 +26,12 @@ SUPPORTED_DISTROS = { class Package(object): - def __init__(self, name, version, path, include_src=True): + def __init__(self, name, version, path, include_src=True, overwrite=False): self.name = name self.version = version self.path = path self.include_src = include_src + self.overwrite = overwrite def build(self): distro = platform.linux_distribution()[0] @@ -39,6 +40,8 @@ class Package(object): distro) target = SUPPORTED_DISTROS[distro] + if self.overwrite: + overwrite = '-f' # not wrapping in a try block - handled by caller - execute("fpm -s dir -t %s -n %s -v %s %s" % - (target, self.name, self.version, self.path)) + execute("fpm %s -s dir -t %s -n %s -v %s %s" % + (overwrite, target, self.name, self.version, self.path)) diff --git a/giftwrap/settings.py b/giftwrap/settings.py index 612a42b..9b6cf60 100644 --- a/giftwrap/settings.py +++ b/giftwrap/settings.py @@ -14,6 +14,8 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations +DEFAULT_BUILD_TYPE = 'package' + class Settings(object): @@ -22,14 +24,17 @@ class Settings(object): 'base_path': '/opt/openstack' } - def __init__(self, package_name_format=None, version=None, - base_path=None, all_in_one=False): + def __init__(self, build_type=DEFAULT_BUILD_TYPE, + package_name_format=None, version=None, + base_path=None, all_in_one=False, force_overwrite=False): if not version: raise Exception("'version' is a required settings") + self.build_type = build_type self._package_name_format = package_name_format self.version = version self._base_path = base_path self.all_in_one = all_in_one + self.force_overwrite = force_overwrite @property def package_name_format(self): diff --git a/giftwrap/shell.py b/giftwrap/shell.py index 47ea757..407cfb4 100644 --- a/giftwrap/shell.py +++ b/giftwrap/shell.py @@ -18,8 +18,9 @@ import argparse import logging import sys +import giftwrap.builder + from giftwrap.build_spec import BuildSpec -from giftwrap.builder import Builder from giftwrap.color import ColorStreamHandler LOG = logging.getLogger(__name__) @@ -44,7 +45,7 @@ def build(args): manifest = fh.read() buildspec = BuildSpec(manifest) - builder = Builder(buildspec) + builder = giftwrap.builder.create_builder(buildspec) builder.build() except Exception as e: LOG.exception("Oops something went wrong: %s", e) diff --git a/giftwrap/templates/Dockerfile.jinja2 b/giftwrap/templates/Dockerfile.jinja2 new file mode 100644 index 0000000..bb90490 --- /dev/null +++ b/giftwrap/templates/Dockerfile.jinja2 @@ -0,0 +1,12 @@ +FROM {{ base_image }} +{% if maintainer -%} +MAINTAINER {{ maintainer }} +{% endif %} + +{% for k,v in envvars.iteritems() -%} +ENV {{ k }} {{ v }} +{% endfor -%} + +{% for command in commands -%} +RUN {{ command }} +{% endfor %} diff --git a/giftwrap/util.py b/giftwrap/util.py index 0fcd490..31e925d 100644 --- a/giftwrap/util.py +++ b/giftwrap/util.py @@ -36,7 +36,7 @@ def execute(command, cwd=None, exit=0): if cwd: original_dir = os.getcwd() os.chdir(cwd) - LOG.debug("Changed directory to %s", cwd) + LOG.info("Changed directory to %s", cwd) LOG.info("Running: '%s'", command) process = subprocess.Popen(command, @@ -48,12 +48,12 @@ def execute(command, cwd=None, exit=0): (out, err) = process.communicate() exitcode = process.wait() - LOG.debug("Command exitted with rc: %s; STDOUT: %s; STDERR: %s" % + LOG.info("Command exitted with rc: %s; STDOUT: %s; STDERR: %s" % (exitcode, out, err)) if cwd: os.chdir(original_dir) - LOG.debug("Changed directory back to %s", original_dir) + LOG.info("Changed directory back to %s", original_dir) if exitcode != exit: raise Exception("Failed to run '%s': rc: %d, out: '%s', err: '%s'" % diff --git a/requirements.txt b/requirements.txt index 0de39f1..b39fff1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,4 @@ pyyaml jinja2 requests pygerrit +docker-py diff --git a/sample.yml b/sample.yml index fc8b2db..60e9fe3 100644 --- a/sample.yml +++ b/sample.yml @@ -1,22 +1,23 @@ --- settings: - package_name_format: 'giftwrap-openstack-{{project.name}}' - version: '1.0-icehouse' + package_name_format: 'openstack-{{project.name}}-{{version}}' + build_type: docker + version: '9.0-bbc5' base_path: '/opt/giftwrap/openstack-{{version}}/' projects: - - name: keystone - gitref: stable/icehouse +# - name: keystone +# gitref: stable/icehouse - name: glance gitref: stable/icehouse - - name: nova - gitref: stable/icehouse - - name: cinder - gitref: stable/icehouse - - name: horizon - gitref: stable/icehouse +# - name: nova +# gitref: stable/icehouse +# - name: cinder +# gitref: stable/icehouse +# - name: horizon +# gitref: stable/icehouse - - name: python-keystoneclient - - name: python-glanceclient - - name: python-novaclient - - name: python-cinderclient +# - name: python-keystoneclient +# - name: python-glanceclient +# - name: python-novaclient +# - name: python-cinderclient diff --git a/setup.cfg b/setup.cfg index 8fb9043..6d60213 100644 --- a/setup.cfg +++ b/setup.cfg @@ -7,6 +7,7 @@ description-file = author = John Dewey author-email = jodewey@cisco.com home-page = https://github.com/cloudcadre/giftwrap +include_package_data = True classifier = Intended Audience :: Information Technology Intended Audience :: Developers @@ -25,6 +26,8 @@ console_scripts = [files] packages = giftwrap +extra_files = + giftwrap/templates/Dockerfile.jinja2 [build_sphinx] all_files = 1 diff --git a/tox.ini b/tox.ini index 26fb756..43967ca 100644 --- a/tox.ini +++ b/tox.ini @@ -17,7 +17,7 @@ downloadcache = {toxworkdir}/_download [testenv:pep8] commands = - flake8 {posargs} + flake8 {posargs} {toxinidir}/giftwrap [testenv:venv] commands = {posargs}