Merge branch 'codedrop' of github.com:craigtracey/giftwrap into craigtracey-codedrop
Conflicts: giftwrap/util.py requirements.txt
This commit is contained in:
John Dewey
commit
b7b3f93231
|
|
@ -7,3 +7,4 @@ ChangeLog
|
|||
build
|
||||
pbr*.egg
|
||||
*.pyc
|
||||
*.sw?
|
||||
|
|
|
|||
|
|
@ -16,27 +16,36 @@
|
|||
|
||||
import yaml
|
||||
|
||||
DEFAULT_PROJECT_PATH = '/opt/openstack'
|
||||
from giftwrap.openstack_project import OpenstackProject
|
||||
from giftwrap.settings import Settings
|
||||
from jinja2 import Template
|
||||
|
||||
|
||||
class BuildSpec(object):
|
||||
|
||||
def __init__(self, manifest):
|
||||
self._spec = yaml.load(manifest)
|
||||
self._project_path = None
|
||||
self._projects = None
|
||||
def __init__(self, manifest, version=None, templatevars=None):
|
||||
self._manifest = self._render_manifest(manifest, version, templatevars)
|
||||
self.projects = self._render_projects()
|
||||
self.settings = self._render_settings()
|
||||
|
||||
@property
|
||||
def project_path(self):
|
||||
if not self._project_path:
|
||||
if 'project_path' in self._spec.keys():
|
||||
self._project_path = self._spec['project_path']
|
||||
else:
|
||||
self._project_path = DEFAULT_PROJECT_PATH
|
||||
return self._project_path
|
||||
def _render_manifest(self, manifest, version=None, templatevars=None):
|
||||
manifestvars = {}
|
||||
if templatevars:
|
||||
manifestvars = yaml.load(templatevars)
|
||||
|
||||
@property
|
||||
def projects(self):
|
||||
if 'projects' in self._spec.keys() and not self._projects:
|
||||
self._projects = self._spec['projects']
|
||||
return self._projects
|
||||
if version:
|
||||
manifestvars['version'] = version
|
||||
|
||||
template = Template(manifest)
|
||||
manifest = template.render(manifestvars)
|
||||
return yaml.load(manifest)
|
||||
|
||||
def _render_projects(self):
|
||||
projects = []
|
||||
if 'projects' in self._manifest:
|
||||
for project in self._manifest['projects']:
|
||||
projects.append(OpenstackProject.factory(project))
|
||||
return projects
|
||||
|
||||
def _render_settings(self):
|
||||
return Settings.factory(self._manifest)
|
||||
|
|
|
|||
|
|
@ -17,7 +17,11 @@
|
|||
import os
|
||||
import sys
|
||||
|
||||
from giftwrap.gerrit import GerritReview
|
||||
from giftwrap.openstack_git_repo import OpenstackGitRepo
|
||||
from giftwrap.openstack_project import OpenstackProject
|
||||
from giftwrap.shell import LOG
|
||||
from giftwrap.util import execute
|
||||
|
||||
|
||||
class Builder(object):
|
||||
|
|
@ -29,7 +33,24 @@ class Builder(object):
|
|||
""" this is where all the magic happens """
|
||||
|
||||
try:
|
||||
os.makedirs(self._spec.project_path)
|
||||
spec = self._spec
|
||||
base_path = spec.settings.base_path
|
||||
version = spec.settings.version
|
||||
|
||||
os.makedirs(base_path)
|
||||
for project in self._spec.projects:
|
||||
project_git_path = os.path.join(base_path, project.name)
|
||||
repo = OpenstackGitRepo(project.giturl, project.ref)
|
||||
repo.clone(project_git_path)
|
||||
|
||||
review = GerritReview(repo.change_id, project.project_path)
|
||||
print "Cloned %s with change_id of: %s" % (project.name, repo.change_id)
|
||||
print "...with pip dependencies of:"
|
||||
print review.build_pip_dependencies(string=True)
|
||||
|
||||
#execute('python tools/install_venv.py', cwd=project_git_path)
|
||||
#deps_string = " -d ".join(deps)
|
||||
#execute("fpm -s dir -t deb -n foobar -d %s /tmp/openstack" % deps_string)
|
||||
except Exception as e:
|
||||
LOG.fatal("Oops. Something went wrong. Error was:\n%s", e)
|
||||
LOG.exception("Oops. Something went wrong. Error was:\n%s", e)
|
||||
sys.exit(-1)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,121 @@
|
|||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2014, Craig Tracey <craigtracey@gmail.com>
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
|
||||
import json
|
||||
import re
|
||||
import requests
|
||||
import sys
|
||||
|
||||
from pygerrit.rest import GerritRestAPI
|
||||
|
||||
DEFAULT_GERRIT_URL = 'https://review.openstack.org'
|
||||
|
||||
|
||||
class GerritReview(object):
|
||||
|
||||
def __init__(self, changeid, project, gerrit_url=DEFAULT_GERRIT_URL):
|
||||
self.changeid = changeid
|
||||
self.project = project
|
||||
self._gerrit_url = gerrit_url
|
||||
self._restclient = None
|
||||
|
||||
def build_pip_dependencies(self, py26=False, py27=True, string=False):
|
||||
url = self._get_gate_build_log_url(py26, py27)
|
||||
response = requests.get(url)
|
||||
|
||||
if response.status_code != 200:
|
||||
raise Exception("Unable to get console log at %s. Error: %d" %
|
||||
(url, response.status_code))
|
||||
|
||||
log = response.text.encode('utf-8')
|
||||
|
||||
freeze_found = False
|
||||
dependencies = []
|
||||
for line in log.split('\n'):
|
||||
line = re.sub('.*\|\s*', '', line)
|
||||
if not freeze_found:
|
||||
if line.endswith("pip freeze"):
|
||||
freeze_found = True
|
||||
continue
|
||||
elif re.match('[\w\-]+==.+', line) or line.startswith('-e'):
|
||||
dependencies.append(line)
|
||||
|
||||
if string:
|
||||
return ('\n').join(dependencies)
|
||||
return dependencies
|
||||
|
||||
def _get_rest_client(self):
|
||||
if not self._restclient:
|
||||
self._restclient = GerritRestAPI(url=self._gerrit_url)
|
||||
return self._restclient
|
||||
|
||||
def _get_review_detail(self):
|
||||
""" get review details for a given change ID """
|
||||
restclient = self._get_rest_client()
|
||||
url = "/changes/?q=%s" % self.changeid
|
||||
changes = restclient.get(url)
|
||||
|
||||
change = None
|
||||
for c in changes:
|
||||
if c['project'] == self.project:
|
||||
change = c
|
||||
break
|
||||
|
||||
if not change:
|
||||
raise Exception("could not find change with ID: %s" % self.changeid)
|
||||
|
||||
detail = restclient.get("/changes/%s/detail" % change['id'])
|
||||
return detail
|
||||
|
||||
def _get_reveiew_messages(self):
|
||||
details = self._get_review_detail()
|
||||
return details['messages']
|
||||
|
||||
def _get_gate_build_log_url(self, py26, py27):
|
||||
messages = self._get_reveiew_messages()
|
||||
messages.reverse()
|
||||
|
||||
mergemsg = None
|
||||
for message in messages:
|
||||
msgtext = message['message']
|
||||
if re.search('Patch Set \d+: Verified', msgtext):
|
||||
mergemsg = msgtext
|
||||
break
|
||||
|
||||
gate_info = self._parse_merge_message(mergemsg)
|
||||
url = None
|
||||
for gate in gate_info:
|
||||
if py26 and re.match('gate\-.+\-python26', gate['name']):
|
||||
url = gate['url']
|
||||
if py27 and re.match('gate\-.+\-python27', gate['name']):
|
||||
url = gate['url']
|
||||
|
||||
if url:
|
||||
return "%s/console.html.gz" % url
|
||||
return url
|
||||
|
||||
def _parse_merge_message(self, msg):
|
||||
""" a function that parses a successful gate gerrit message """
|
||||
gate_info = []
|
||||
for line in msg.split('\n'):
|
||||
parts = re.split('\s+', line)
|
||||
if parts[0] == '-':
|
||||
gate = {}
|
||||
gate['name'] = parts[1]
|
||||
gate['url'] = parts[2]
|
||||
gate['result'] = parts[4]
|
||||
gate_info.append(gate)
|
||||
return gate_info
|
||||
|
|
@ -0,0 +1,82 @@
|
|||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2014, Craig Tracey <craigtracey@gmail.com>
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
|
||||
import datetime
|
||||
import re
|
||||
|
||||
from git import Repo
|
||||
|
||||
|
||||
class OpenstackGitRepo(object):
|
||||
|
||||
def __init__(self, url, ref='master'):
|
||||
self.url = url
|
||||
self.ref = ref
|
||||
self._repo = None
|
||||
self._head = None
|
||||
self._change_id = None
|
||||
self._committed_date = None
|
||||
|
||||
@property
|
||||
def cloned(self):
|
||||
return not self._repo == None
|
||||
|
||||
@property
|
||||
def head(self):
|
||||
if not self._head and self._repo:
|
||||
self._head = self._repo.head.commit.hexsha
|
||||
return self._head
|
||||
|
||||
@property
|
||||
def change_id(self):
|
||||
if not self._change_id and self._repo:
|
||||
for commit in self._repo.iter_commits():
|
||||
match = re.search('Change-Id:\s*(I\w+)', commit.message)
|
||||
if match:
|
||||
self._change_id = match.group(1)
|
||||
break
|
||||
return self._change_id
|
||||
|
||||
@property
|
||||
def committed_date(self):
|
||||
if not self._committed_date and self._repo:
|
||||
self._committed_date = self._repo.head.commit.committed_date
|
||||
return self._committed_date
|
||||
|
||||
def _invalidate_attrs(self):
|
||||
self._head = None
|
||||
self._change_id = None
|
||||
self._committed_date = None
|
||||
|
||||
def clone(self, outdir):
|
||||
self._repo = Repo.clone_from(self.url, outdir)
|
||||
git = self._repo.git
|
||||
git.checkout(self.ref)
|
||||
self._invalidate_attrs()
|
||||
|
||||
def reset_to_date(self, date):
|
||||
if self._repo:
|
||||
commit_date_sha = None
|
||||
for commit in self._repo.iter_commits():
|
||||
if commit.committed_date >= date:
|
||||
commit_date_sha = commit.hexsha
|
||||
elif commit.committed_date < date:
|
||||
break
|
||||
if not commit_date_sha:
|
||||
raise Exception("Unable to find commit for date %s",
|
||||
datetime.datetime.fromtimestamp(date))
|
||||
git = self._repo.git
|
||||
git.checkout(commit_date_sha)
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2014, Craig Tracey <craigtracey@gmail.com>
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
|
||||
|
||||
OPENSTACK_PROJECTS = [
|
||||
{
|
||||
'name': 'keystone',
|
||||
'project_path': 'openstack/keystone',
|
||||
'giturl': 'https://github.com/openstack/keystone.git'
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
class OpenstackProject(object):
|
||||
|
||||
def __init__(self, name, ref=None, giturl=None):
|
||||
if not name in OpenstackProject.get_project_names():
|
||||
raise Exception("'%s' is not a supported OpenStack project name" %
|
||||
name)
|
||||
self.name = name
|
||||
self._project = [p for p in OPENSTACK_PROJECTS if p['name'] == name][0]
|
||||
self.ref = ref if ref else 'master'
|
||||
self.project_path = self._project['project_path']
|
||||
if not giturl:
|
||||
self.giturl = OpenstackProject.get_project_giturl(name)
|
||||
else:
|
||||
self.giturl = giturl
|
||||
|
||||
@staticmethod
|
||||
def factory(project_dict):
|
||||
name = project_dict.get('name', None)
|
||||
ref = project_dict.get('ref', None)
|
||||
giturl = project_dict.get('giturl', None)
|
||||
return OpenstackProject(name, ref, giturl)
|
||||
|
||||
@staticmethod
|
||||
def get_project_names():
|
||||
return [n['name'] for n in OPENSTACK_PROJECTS]
|
||||
|
||||
@staticmethod
|
||||
def get_project_giturl(name):
|
||||
for project in OPENSTACK_PROJECTS:
|
||||
if name == project['name']:
|
||||
return project['giturl']
|
||||
return None
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
||||
|
||||
# Copyright 2014, Craig Tracey <craigtracey@gmail.com>
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
|
||||
|
||||
class Settings(object):
|
||||
|
||||
DEFAULT_BASE_PATH = '/opt/openstack'
|
||||
|
||||
def __init__(self, version, base_path):
|
||||
self._validate_settings(version, base_path)
|
||||
self.version = version
|
||||
self.base_path = base_path
|
||||
|
||||
def _validate_settings(self, version, base_path):
|
||||
if not version:
|
||||
raise Exception("You must provide a version")
|
||||
|
||||
@staticmethod
|
||||
def factory(settings_dict):
|
||||
version = settings_dict.get('version', None)
|
||||
base_path = settings_dict.get('base_path', Settings.DEFAULT_BASE_PATH)
|
||||
return Settings(version, base_path)
|
||||
|
|
@ -17,31 +17,37 @@
|
|||
import argparse
|
||||
import logging
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
from giftwrap.build_spec import BuildSpec
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
log_handler = logging.StreamHandler()
|
||||
LOG.addHandler(log_handler)
|
||||
LOG.setLevel(logging.DEBUG)
|
||||
|
||||
from giftwrap.builder import Builder
|
||||
from giftwrap.settings import Settings
|
||||
|
||||
|
||||
def build(args):
|
||||
""" the entry point for all build subcommand tasks """
|
||||
if not args.manifest:
|
||||
LOG.fatal("build command requires a manifest")
|
||||
sys.exit(-1)
|
||||
|
||||
try:
|
||||
manifest = None
|
||||
templatevars = None
|
||||
|
||||
with open(args.manifest, 'r') as fh:
|
||||
manifest = fh.read()
|
||||
buildspec = BuildSpec(manifest)
|
||||
|
||||
if args.templatevars:
|
||||
with open(args.templatevars, 'r') as fh:
|
||||
templatevars = fh.read()
|
||||
|
||||
buildspec = BuildSpec(manifest, args.version, templatevars)
|
||||
builder = Builder(buildspec)
|
||||
builder.build()
|
||||
except Exception as e:
|
||||
LOG.fatal("Unable to parse manifest %s. Error: %s", args.manifest, e)
|
||||
LOG.exception("Unable to parse manifest. Error: %s", e)
|
||||
sys.exit(-1)
|
||||
|
||||
|
||||
|
|
@ -54,7 +60,11 @@ def main():
|
|||
help='additional help')
|
||||
build_subcmd = subparsers.add_parser('build',
|
||||
description='build giftwrap packages')
|
||||
build_subcmd.add_argument('-m', '--manifest')
|
||||
build_subcmd.add_argument('-m', '--manifest', required=True)
|
||||
build_subcmd.add_argument('-a', '--allinone', action='store_true')
|
||||
build_subcmd.add_argument('-v', '--version')
|
||||
build_subcmd.add_argument('-s', '--source', action='store_true')
|
||||
build_subcmd.add_argument('-t', '--templatevars')
|
||||
build_subcmd.set_defaults(func=build)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
|
|
|||
|
|
@ -19,17 +19,27 @@ import os
|
|||
import subprocess
|
||||
|
||||
from git import Repo
|
||||
|
||||
import giturlparse
|
||||
|
||||
from giftwrap.shell import LOG
|
||||
|
||||
def execute(command):
|
||||
|
||||
def execute(command, cwd=None):
|
||||
"""
|
||||
Executes a command in a subprocess. Returns a tuple of
|
||||
(exitcode, out, err).
|
||||
|
||||
:param command: Command string to execute.
|
||||
:param cwd: Directory to execute from.
|
||||
"""
|
||||
|
||||
original_dir = None
|
||||
if cwd:
|
||||
original_dir = os.getcwd()
|
||||
os.chdir(cwd)
|
||||
LOG.info("Changed directory to %s", cwd)
|
||||
|
||||
LOG.info("Running %s", command)
|
||||
process = subprocess.Popen(command,
|
||||
cwd=os.getcwd(),
|
||||
stdin=subprocess.PIPE,
|
||||
|
|
@ -39,6 +49,13 @@ def execute(command):
|
|||
(out, err) = process.communicate()
|
||||
exitcode = process.wait()
|
||||
|
||||
LOG.debug("Command exitted with rc: %s; STDOUT: %s; STDERR: %s" %
|
||||
(exitcode, out, err))
|
||||
|
||||
if cwd:
|
||||
os.chdir(original_dir)
|
||||
LOG.info("Changed directory back to %s", original_dir)
|
||||
|
||||
return exitcode, out, err
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,8 @@
|
|||
argparse
|
||||
pbr
|
||||
GitPython==0.3.2.RC1
|
||||
giturlparse.py
|
||||
pyyaml
|
||||
jinja2
|
||||
requests
|
||||
pygerrit
|
||||
|
|
|
|||
|
|
@ -1,9 +1,7 @@
|
|||
---
|
||||
package_path: '/opt/openstack/'
|
||||
version: 1.2
|
||||
base_path: '/tmp/openstack-test'
|
||||
|
||||
projects:
|
||||
- name: nova
|
||||
url: https://github.com/openstack/nova
|
||||
|
||||
- name: keystone
|
||||
url: https://github.com/openstack/keystone
|
||||
ref: stable/havana
|
||||
|
|
|
|||
Loading…
Reference in New Issue