x
/
giftwrap
Code Issues Proposed changes

Refactor with builder support 

Refactor the builders to so that a user may specify one of 2 paths:
system packages or docker containers. This change also encompasses a
bunch of other minor changes.  This is being checked in in order to
get some minimally viable changes upstream.
This commit is contained in:
Craig Tracey 2014-11-14 18:44:36 -05:00
parent 9142754368
commit 37842bcc97
16 changed files with 319 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -8,3 +8,4 @@ build
pbr*.egg
*.pyc
*.sw?
*.deb

54
giftwrap/builder.py
View File

@ -15,13 +15,6 @@
# License for the specific language governing permissions and limitations
import logging
import os
import sys
from giftwrap.gerrit import GerritReview
from giftwrap.openstack_git_repo import OpenstackGitRepo
from giftwrap.package import Package
from giftwrap.util import execute
LOG = logging.getLogger(__name__)
@ -30,39 +23,26 @@ class Builder(object):
def __init__(self, spec):
self._spec = spec
self.settings = spec.settings
def _build(self):
raise NotImplementedError()
def _validate_settings(self):
raise NotImplementedError()
def build(self):
""" this is where all the magic happens """
self._validate_settings()
self._build()
try:
spec = self._spec
for project in self._spec.projects:
LOG.info("Beginning to build '%s'", project.name)
os.makedirs(project.install_path)
LOG.info("Fetching source code for '%s'", project.name)
repo = OpenstackGitRepo(project.giturl, project.gitref)
repo.clone(project.install_path)
review = GerritReview(repo.change_id, project.git_path)
from giftwrap.builders.package_builder import PackageBuilder
from giftwrap.builders.docker_builder import DockerBuilder
LOG.info("Creating the virtualenv for '%s'", project.name)
execute(project.venv_command, project.install_path)
LOG.info("Installing '%s' pip dependencies to the virtualenv",
project.name)
execute(project.install_command %
review.build_pip_dependencies(string=True),
project.install_path)
LOG.info("Installing '%s' to the virtualenv", project.name)
execute(".venv/bin/python setup.py install",
project.install_path)
if not spec.settings.all_in_one:
pkg = Package(project.package_name, project.version,
project.install_path, True)
pkg.build()
except Exception as e:
LOG.exception("Oops. Something went wrong. Error was:\n%s", e)
sys.exit(-1)
def create_builder(spec):
if spec.settings.build_type == 'package':
return PackageBuilder(spec)
elif spec.settings.build_type == 'docker':
return DockerBuilder(spec)
raise Exception("Unknown build_type: '%s'", spec.settings.build_type)

0
giftwrap/builders/__init__.py Normal file
View File

169
giftwrap/builders/docker_builder.py Normal file
View File

@ -0,0 +1,169 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2014, Craig Tracey <craigtracey@gmail.com>
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
import docker
import json
import logging
import os
import re
import tempfile
from giftwrap.builder import Builder
LOG = logging.getLogger(__name__)
import jinja2
DEFAULT_TEMPLATE_FILE = os.path.join(os.path.dirname(
os.path.dirname(__file__)),
'templates/Dockerfile.jinja2')
APT_REQUIRED_PACKAGES = [
'libffi-dev',
'libxml2-dev',
'libxslt1-dev',
'git',
'wget',
'curl',
'libldap2-dev',
'libsasl2-dev',
'libssl-dev',
'python-dev',
'libmysqlclient-dev',
'python-virtualenv',
'python-pip',
'build-essential'
]
class DockerBuilder(Builder):
def __init__(self, spec):
self.template = DEFAULT_TEMPLATE_FILE
self.base_image = 'ubuntu:12.04'
self.maintainer = 'maintainer@example.com'
self.envvars = {'DEBIAN_FRONTEND': 'noninteractive'}
self._paths = []
super(DockerBuilder, self).__init__(spec)
def _validate_settings(self):
if not self.settings.all_in_one:
LOG.warn("The Docker builder does not support all-in-one")
def _get_prep_commands(self):
commands = []
commands.append('apt-get update && apt-get install -y %s' %
' '.join(APT_REQUIRED_PACKAGES))
return commands
def _get_build_commands(self, src_path):
commands = []
commands.append('mkdir -p %s' % src_path)
for project in self._spec.projects:
if project.system_dependencies:
commands.append('apt-get update && apt-get install -y %s' %
' '.join(project.system_dependencies))
project_src_path = os.path.join(src_path, project.name)
commands.append('git clone %s -b %s %s' % (project.giturl,
project.gitref,
project_src_path))
commands.append('mkdir -p %s' %
os.path.dirname(project.install_path))
commands.append('virtualenv --system-site-packages %s' %
project.install_path)
project_bin_path = os.path.join(project.install_path, 'bin')
self._paths.append(project_bin_path)
venv_python_path = os.path.join(project_bin_path, 'python')
venv_pip_path = os.path.join(project_bin_path, 'pip')
if project.pip_dependencies:
commands.append("%s install %s" % (venv_pip_path,
' '.join(project.pip_dependencies)))
commands.append('cd %s && %s setup.py install && cd -' %
(project_src_path, venv_python_path))
commands.append("%s install pbr" % venv_pip_path)
return commands
def _get_cleanup_commands(self, src_path):
commands = []
commands.append('rm -rf %s' % src_path)
return commands
def _set_path(self):
path = ":".join(self._paths)
self.envvars['PATH'] = "%s:$PATH" % path
def _render_dockerfile(self, extra_vars):
template_vars = self.__dict__.update(extra_vars)
template_loader = jinja2.FileSystemLoader(searchpath='/')
template_env = jinja2.Environment(loader=template_loader)
template = template_env.get_template(self.template)
return template.render(template_vars)
def _build(self):
src_path = '/tmp/build'
commands = self._get_prep_commands()
commands += self._get_build_commands(src_path)
commands += self._get_cleanup_commands(src_path)
self._set_path()
dockerfile_contents = self._render_dockerfile(locals())
tempdir = tempfile.mkdtemp()
dockerfile = os.path.join(tempdir, 'Dockerfile')
with open(dockerfile, "w") as w:
w.write(dockerfile_contents)
docker_client = docker.Client(base_url='unix://var/run/docker.sock',
version='1.10', timeout=10)
build_result = docker_client.build(path=tempdir, stream=True,
tag='openstack-9.0:bbc6')
for line in build_result:
LOG.info(line.strip())
# I borrowed this from docker/stackbrew, should cull it down
# to be more sane.
def _parse_result(self, build_result):
build_success_re = r'^Successfully built ([a-f0-9]+)\n$'
if isinstance(build_result, tuple):
img_id, logs = build_result
return img_id, logs
else:
lines = [line for line in build_result]
try:
parsed_lines = [json.loads(e).get('stream', '') for e in lines]
except ValueError:
# sometimes all the data is sent on a single line ????
#
# ValueError: Extra data: line 1 column 87 - line 1 column
# 33268 (char 86 - 33267)
line = lines[0]
# This ONLY works because every line is formatted as
# {"stream": STRING}
parsed_lines = [
json.loads(obj).get('stream', '') for obj in
re.findall('{\s*"stream"\s*:\s*"[^"]*"\s*}', line)
]
for line in parsed_lines:
match = re.match(build_success_re, line)
if match:
return match.group(1), parsed_lines
return None, lines

74
giftwrap/builders/package_builder.py Normal file
View File

@ -0,0 +1,74 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2014, Craig Tracey <craigtracey@gmail.com>
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
import logging
import os
import shutil
from giftwrap.gerrit import GerritReview
from giftwrap.openstack_git_repo import OpenstackGitRepo
from giftwrap.package import Package
from giftwrap.util import execute
LOG = logging.getLogger(__name__)
from giftwrap.builder import Builder
class PackageBuilder(Builder):
def __init__(self, spec):
self._all_in_one = False
super(PackageBuilder, self).__init__(spec)
def _validate_settings(self):
pass
def _build(self):
spec = self._spec
for project in self._spec.projects:
LOG.info("Beginning to build '%s'", project.name)
if (os.path.exists(project.install_path) and
spec.settings.force_overwrite):
LOG.info("force_overwrite is set, so removing "
"existing path '%s'" % project.install_path)
shutil.rmtree(project.install_path)
os.makedirs(project.install_path)
LOG.info("Fetching source code for '%s'", project.name)
repo = OpenstackGitRepo(project.giturl, project.gitref)
repo.clone(project.install_path)
review = GerritReview(repo.change_id, project.git_path)
LOG.info("Creating the virtualenv for '%s'", project.name)
execute(project.venv_command, project.install_path)
LOG.info("Installing '%s' pip dependencies to the virtualenv",
project.name)
execute(project.install_command %
review.build_pip_dependencies(string=True),
project.install_path)
LOG.info("Installing '%s' to the virtualenv", project.name)
execute(".venv/bin/python setup.py install",
project.install_path)
if not spec.settings.all_in_one:
pkg = Package(project.package_name, project.version,
project.install_path, True,
spec.settings.force_overwrite)
pkg.build()

6
giftwrap/openstack_git_repo.py
View File

@ -65,7 +65,7 @@ class OpenstackGitRepo(object):
self._committed_date = None
def clone(self, outdir):
LOG.debug("Cloning '%s' to '%s'", self.url, outdir)
LOG.info("Cloning '%s' to '%s'", self.url, outdir)
self._repo = Repo.clone_from(self.url, outdir)
git = self._repo.git
git.checkout(self.ref)
@ -83,6 +83,6 @@ class OpenstackGitRepo(object):
raise Exception("Unable to find commit for date %s",
datetime.datetime.fromtimestamp(date))
git = self._repo.git
LOG.debug("Reset repo '%s' to commit at '%s'", self.url,
time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(date)))
LOG.info("Reset repo '%s' to commit at '%s'", self.url,
time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(date)))
git.checkout(commit_date_sha)

5
giftwrap/openstack_project.py
View File

@ -34,7 +34,8 @@ class OpenstackProject(object):
def __init__(self, settings, name, version=None, gitref=None, giturl=None,
venv_command=None, install_command=None, install_path=None,
package_name=None, stackforge=False):
package_name=None, stackforge=False, system_dependencies=[],
pip_dependencies=[]):
self._settings = settings
self.name = name
self._version = version
@ -46,6 +47,8 @@ class OpenstackProject(object):
self._package_name = package_name
self.stackforge = stackforge
self._git_path = None
self.system_dependencies = system_dependencies
self.pip_dependencies = pip_dependencies
@property
def version(self):

9
giftwrap/package.py
View File

@ -26,11 +26,12 @@ SUPPORTED_DISTROS = {
class Package(object):
def __init__(self, name, version, path, include_src=True):
def __init__(self, name, version, path, include_src=True, overwrite=False):
self.name = name
self.version = version
self.path = path
self.include_src = include_src
self.overwrite = overwrite
def build(self):
distro = platform.linux_distribution()[0]
@ -39,6 +40,8 @@ class Package(object):
distro)
target = SUPPORTED_DISTROS[distro]
if self.overwrite:
overwrite = '-f'
# not wrapping in a try block - handled by caller
execute("fpm -s dir -t %s -n %s -v %s %s" %
(target, self.name, self.version, self.path))
execute("fpm %s -s dir -t %s -n %s -v %s %s" %
(overwrite, target, self.name, self.version, self.path))

9
giftwrap/settings.py
View File

@ -14,6 +14,8 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
DEFAULT_BUILD_TYPE = 'package'
class Settings(object):
@ -22,14 +24,17 @@ class Settings(object):
'base_path': '/opt/openstack'
}
def __init__(self, package_name_format=None, version=None,
base_path=None, all_in_one=False):
def __init__(self, build_type=DEFAULT_BUILD_TYPE,
package_name_format=None, version=None,
base_path=None, all_in_one=False, force_overwrite=False):
if not version:
raise Exception("'version' is a required settings")
self.build_type = build_type
self._package_name_format = package_name_format
self.version = version
self._base_path = base_path
self.all_in_one = all_in_one
self.force_overwrite = force_overwrite
@property
def package_name_format(self):

5
giftwrap/shell.py
View File

@ -18,8 +18,9 @@ import argparse
import logging
import sys
import giftwrap.builder
from giftwrap.build_spec import BuildSpec
from giftwrap.builder import Builder
from giftwrap.color import ColorStreamHandler
LOG = logging.getLogger(__name__)
@ -44,7 +45,7 @@ def build(args):
manifest = fh.read()
buildspec = BuildSpec(manifest)
builder = Builder(buildspec)
builder = giftwrap.builder.create_builder(buildspec)
builder.build()
except Exception as e:
LOG.exception("Oops something went wrong: %s", e)

12
giftwrap/templates/Dockerfile.jinja2 Normal file
View File

@ -0,0 +1,12 @@
FROM {{ base_image }}
{% if maintainer -%}
MAINTAINER {{ maintainer }}
{% endif %}
{% for k,v in envvars.iteritems() -%}
ENV {{ k }} {{ v }}
{% endfor -%}
{% for command in commands -%}
RUN {{ command }}
{% endfor %}

6
giftwrap/util.py
View File

@ -36,7 +36,7 @@ def execute(command, cwd=None, exit=0):
if cwd:
original_dir = os.getcwd()
os.chdir(cwd)
LOG.debug("Changed directory to %s", cwd)
LOG.info("Changed directory to %s", cwd)
LOG.info("Running: '%s'", command)
process = subprocess.Popen(command,
@ -48,12 +48,12 @@ def execute(command, cwd=None, exit=0):
(out, err) = process.communicate()
exitcode = process.wait()
LOG.debug("Command exitted with rc: %s; STDOUT: %s; STDERR: %s" %
LOG.info("Command exitted with rc: %s; STDOUT: %s; STDERR: %s" %
(exitcode, out, err))
if cwd:
os.chdir(original_dir)
LOG.debug("Changed directory back to %s", original_dir)
LOG.info("Changed directory back to %s", original_dir)
if exitcode != exit:
raise Exception("Failed to run '%s': rc: %d, out: '%s', err: '%s'" %

1
requirements.txt
View File

@ -7,3 +7,4 @@ pyyaml
jinja2
requests
pygerrit
docker-py

29
sample.yml
View File

@ -1,22 +1,23 @@
---
settings:
package_name_format: 'giftwrap-openstack-{{project.name}}'
version: '1.0-icehouse'
package_name_format: 'openstack-{{project.name}}-{{version}}'
build_type: docker
version: '9.0-bbc5'
base_path: '/opt/giftwrap/openstack-{{version}}/'
projects:
- name: keystone
gitref: stable/icehouse
# - name: keystone
# gitref: stable/icehouse
- name: glance
gitref: stable/icehouse
- name: nova
gitref: stable/icehouse
- name: cinder
gitref: stable/icehouse
- name: horizon
gitref: stable/icehouse
# - name: nova
# gitref: stable/icehouse
# - name: cinder
# gitref: stable/icehouse
# - name: horizon
# gitref: stable/icehouse
- name: python-keystoneclient
- name: python-glanceclient
- name: python-novaclient
- name: python-cinderclient
# - name: python-keystoneclient
# - name: python-glanceclient
# - name: python-novaclient
# - name: python-cinderclient

3
setup.cfg
View File

@ -7,6 +7,7 @@ description-file =
author = John Dewey
author-email = jodewey@cisco.com
home-page = https://github.com/cloudcadre/giftwrap
include_package_data = True
classifier =
Intended Audience :: Information Technology
Intended Audience :: Developers
@ -25,6 +26,8 @@ console_scripts =
[files]
packages =
giftwrap
extra_files =
giftwrap/templates/Dockerfile.jinja2
[build_sphinx]
all_files = 1

2
tox.ini
View File

@ -17,7 +17,7 @@ downloadcache = {toxworkdir}/_download
[testenv:pep8]
commands =
flake8 {posargs}
flake8 {posargs} {toxinidir}/giftwrap
[testenv:venv]
commands = {posargs}