Overhaul prep-source-repos.
Key changes: - write out a variables file to avoid doing work during devtestrc. - remove the special casing and hard coding of review.o.o and openstack-infra. - update README.
This commit is contained in:
Robert Collins
parent
0c8115b95f
commit
7798e52299
32
README
32
README
|
|
@ -1,24 +1,24 @@
|
|||
This repository is a place for testers of full rack end-to-end TripleO
|
||||
runs to coordinate around reproducible deployments.
|
||||
This repository contains scripts for managing multiple outstanding patches to
|
||||
TripleO (or other gerrit based projects).
|
||||
|
||||
repo_refs.yaml contains the various repos TripleO will clone and allows
|
||||
you to track in-flight Gerrit reviews that need to be pulled down as
|
||||
part of the deployment.
|
||||
- tooling to combine arbitrary unmerged gerrit patches (prep_source_repos)
|
||||
which will also export an rc file with git refs based on the combined
|
||||
branches
|
||||
- a sample config file that we're using (repo_refs.yaml)
|
||||
|
||||
To use:
|
||||
|
||||
Ideally, maintain a local rc file with environment variables specific
|
||||
to your environment. To view which you should be setting, see top of
|
||||
devtestrc.
|
||||
* create a repo_refs.yaml in your TRIPLEO_ROOT (see the one in the root of this
|
||||
repository for inspiration).
|
||||
|
||||
You'll also want to maintain a nodes.json and bm-network.json specific
|
||||
to your node inventory and seed/undercloud network topology. See examples.
|
||||
* add the tripleo-end-to-end bin directory to your path (or symlink the
|
||||
specific scripts into place, or $up to you).
|
||||
|
||||
With your local overrides and nodes.json and bm-network.json in place, simply
|
||||
source devtestrc. This will clone all required repos and patches every time
|
||||
its sourced. To skip this part, set NO_SOURCE_PREP=1. Note without this set,
|
||||
you'll likely lose local changes to the repos upon resourcing.
|
||||
* run prep_source_repos $YOUR\_REFS\_FILE $TRIPLEO\_ROOT to checkout and update
|
||||
the repositories specified by the refs file
|
||||
|
||||
Then you may simply run:
|
||||
* source YOUR_REFS_FILE.variables to configure TripleO scripts to use your
|
||||
freshly integrated branches
|
||||
|
||||
devtest.sh --trash-my-machine --nodes $PATH_TO/nodes.json --bm-networks $PATH_TO/bm-network.json
|
||||
* proceed with any tripleo activies you might have (building images, deploying,
|
||||
etc etc).
|
||||
|
|
|
|||
|
|
@ -1,51 +1,101 @@
|
|||
#!/usr/bin/python
|
||||
|
||||
import yaml
|
||||
import argparse
|
||||
import os.path
|
||||
import re
|
||||
from subprocess import check_call, check_output
|
||||
import os
|
||||
import sys
|
||||
import yaml
|
||||
|
||||
SRC_ROOT = os.getenv('TRIPLEO_ROOT', 'tripleo')
|
||||
CONF = 'repo_refs.yaml'
|
||||
CONF = yaml.load(open(CONF).read())
|
||||
GIT_SERVER = 'http://review.openstack.org'
|
||||
|
||||
if not os.path.isdir(SRC_ROOT):
|
||||
os.mkdir(SRC_ROOT)
|
||||
def normalise_conf(conf):
|
||||
"""generate full paths etc for easy application later."""
|
||||
def make_repos(thing, path_stack):
|
||||
if isinstance(thing, dict):
|
||||
repos = {}
|
||||
for key, subthing in thing.items():
|
||||
path = path_stack + (key,)
|
||||
repos.update(make_repos(subthing, path))
|
||||
return repos
|
||||
elif isinstance(thing, list):
|
||||
path = '/'.join(path_stack)
|
||||
repos = {}
|
||||
for name in thing:
|
||||
if name in repos:
|
||||
raise ValueError("%r defined multiple times" % name)
|
||||
repos[name] = '%s/%s' % (path, name)
|
||||
return repos
|
||||
else:
|
||||
raise ValueError("%r is not a dict or list" % (thing,))
|
||||
conf['repos'] = make_repos(conf['repos'], ())
|
||||
return conf
|
||||
|
||||
for repo in CONF['repos']:
|
||||
rd = os.path.join(SRC_ROOT, repo)
|
||||
if repo in ['tripleo-ci']:
|
||||
org = 'openstack-infra'
|
||||
else:
|
||||
org = 'openstack'
|
||||
|
||||
remote = GIT_SERVER + '/%s/%s' % (org, repo)
|
||||
if os.path.isdir(os.path.join(rd)):
|
||||
check_call(['git', 'checkout', 'master'], cwd=rd)
|
||||
check_call(['git', 'pull'], cwd=rd)
|
||||
else:
|
||||
check_call(['git', 'clone', remote, rd])
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("refs", help="the yaml config file")
|
||||
parser.add_argument("output", help="where to put the downloaded repositories")
|
||||
args = parser.parse_args()
|
||||
SRC_ROOT = os.path.abspath(args.output)
|
||||
with open(args.refs, 'rt') as arg_file:
|
||||
CONF = yaml.safe_load(arg_file.read())
|
||||
CONF = normalise_conf(CONF)
|
||||
|
||||
if repo not in CONF['gerrit_refs']:
|
||||
print 'no refs for %s' % repo
|
||||
continue
|
||||
if not os.path.lexists(SRC_ROOT):
|
||||
os.makedirs(SRC_ROOT)
|
||||
|
||||
refs = CONF['gerrit_refs'][repo]
|
||||
branch_name = 'rollup_' + \
|
||||
'_'.join([r[1] for r in [ref.split('/') for ref in refs]])
|
||||
variables = []
|
||||
|
||||
branches = check_output(['git', 'branch', '-a'], cwd=rd)
|
||||
for repo, remote in CONF['repos'].items():
|
||||
rd = os.path.join(SRC_ROOT, repo)
|
||||
|
||||
if branch_name in branches:
|
||||
print 'Deleting existing branch %s...' % branch_name
|
||||
check_call(['git', 'branch', '-D', branch_name], cwd=rd)
|
||||
if not os.path.isdir(os.path.join(rd)):
|
||||
check_call(['git', 'clone', remote, rd])
|
||||
|
||||
check_call(['git', 'checkout', '-b', branch_name], cwd=rd)
|
||||
check_call(['git', 'branch', '--set-upstream-to=origin/master'], cwd=rd)
|
||||
refs = CONF['gerrit_refs'].get(repo, ())
|
||||
|
||||
for ref in CONF['gerrit_refs'][repo]:
|
||||
_, rev, p = ref.split('/')
|
||||
print 'pulling %s' % (GIT_SERVER + '/#/c/' + rev)
|
||||
print ' '.join(['git', 'pull', remote, ref])
|
||||
check_call(['git', 'pull', '--no-edit', remote, 'refs/changes/' + ref],
|
||||
cwd=rd)
|
||||
git_refs = ['master']
|
||||
for ref in refs:
|
||||
git_refs.append(
|
||||
'+refs/changes/%(ref)s:refs/changes/%(ref)s' % dict(ref=ref))
|
||||
#print ' '.join(['git', 'pull', remote, ref])
|
||||
#check_call(['git', 'pull', '--no-edit', remote, 'refs/changes/' + ref],
|
||||
# cwd=rd)
|
||||
print 'fetching from %s %s' % (remote, git_refs)
|
||||
check_call(['git', 'fetch', remote] + git_refs, cwd=rd)
|
||||
|
||||
if not refs:
|
||||
branch_name = 'master'
|
||||
else:
|
||||
branch_name = 'rollup_' + \
|
||||
'_'.join([r[1] for r in [ref.split('/') for ref in refs]])
|
||||
check_call(['git', 'stash'], cwd=rd)
|
||||
branches = check_output(['git', 'branch', '-a'], cwd=rd)
|
||||
if ' ' + branch_name in branches:
|
||||
print 'Resetting existing branch %s...' % branch_name
|
||||
check_call(['git', 'checkout', branch_name], cwd=rd)
|
||||
check_call(['git', 'reset', '--hard', 'origin/master'], cwd=rd)
|
||||
else:
|
||||
check_call(['git', 'checkout', '-b', branch_name, 'origin/master'], cwd=rd)
|
||||
for ref in refs:
|
||||
check_call(
|
||||
['git', 'merge', '--no-edit', 'refs/changes/%s' % ref], cwd=rd)
|
||||
normalised_repo = re.sub('[^A-Za-z0-9_]', '_', repo)
|
||||
if repo not in CONF['gerrit_refs']:
|
||||
print 'no refs for %s' % repo
|
||||
variables.append((normalised_repo, rd, None))
|
||||
else:
|
||||
variables.append((normalised_repo, rd, branch_name))
|
||||
|
||||
with open(args.refs + '.variables', 'wt') as output:
|
||||
for name, location, ref in variables:
|
||||
output.write('export DIB_REPOTYPE_%s=git\n' % name)
|
||||
output.write('export DIB_REPOLOCATION_%s=%s\n' % (name, location))
|
||||
if ref:
|
||||
output.write('export DIB_REPOREF_%s=%s\n' % (name, ref))
|
||||
else:
|
||||
output.write('unset DIB_REPOREF_%s\n'% name)
|
||||
return 0
|
||||
|
||||
|
||||
sys.exit(main())
|
||||
|
|
|
|||
|
|
@ -1,31 +1,40 @@
|
|||
# hash of repos to download
|
||||
repos:
|
||||
- diskimage-builder
|
||||
- horizon
|
||||
- tripleo-incubator
|
||||
- python-keystoneclient
|
||||
- python-glanceclient
|
||||
- swift
|
||||
- glance
|
||||
- python-neutronclient
|
||||
- python-ceilometerclient
|
||||
- neutron
|
||||
- python-heatclient
|
||||
- python-swiftclient
|
||||
- keystone
|
||||
# each hash becomes a path component
|
||||
http://review.openstack.org:
|
||||
# until we get down to a list
|
||||
openstack:
|
||||
# at which point we're naming a repo and downloading it to disk as ./$name
|
||||
- cinder
|
||||
- python-cinderclient
|
||||
- nova
|
||||
- python-ironicclient
|
||||
- python-novaclient
|
||||
- dib-utils
|
||||
- diskimage-builder
|
||||
- glance
|
||||
- heat
|
||||
- horizon
|
||||
- ironic
|
||||
- keystone
|
||||
- neutron
|
||||
- nova
|
||||
- os-cloud-config
|
||||
- python-ceilometerclient
|
||||
- python-cinderclient
|
||||
- python-glanceclient
|
||||
- python-heatclient
|
||||
- python-ironicclient
|
||||
- python-keystoneclient
|
||||
- python-neutronclient
|
||||
- python-novaclient
|
||||
- python-swiftclient
|
||||
- swift
|
||||
- tripleo-heat-templates
|
||||
- tripleo-image-elements
|
||||
- os-cloud-config
|
||||
- tripleo-incubator
|
||||
openstack-infra:
|
||||
- tripleo-ci
|
||||
- dib-utils
|
||||
|
||||
# A list of patches to pull from gerrit into repos
|
||||
gerrit_refs:
|
||||
# The name of said repo from above.
|
||||
ironic:
|
||||
# Fix tear_down a node with missing info
|
||||
# https://review.openstack.org/#/c/103685/
|
||||
|
|
|
|||
Loading…
Reference in New Issue