zuul
/
zuul
Code Issues Proposed changes

Cloner: use zuul_url always when project set 

In the case of a post-merge pipeline (ie, one that
responds to ref-updated events) this increases the accuracy of
a multi-repo checkout by consulting the zuul merger when
cloning and updating the project in question.  All other projects
are updated from the upstream git url.  Previously all repos would
be cloned and updated from the upstream url.  That is normally
fine, but in some cases they may actually lag behind, but the
merger will have the most current copy of the repo for the project
in question.

This is safe because we now ensure that mergers fully update the
branches and tags of their local copies.

The assertion in test_cache_dir is updated to reflict this change.
Its intent is to verify that the origin in a prepared repo is not
the cache (that would be detrimental since the cache will never
update).  It did this by asserting that the origin was the upstream
url.  However, with this change the origin may be the zuul_url (and
should be for project1 in this test).  So the assertion is changed
to verify that the origin is the zuul url.  A second assertion that
it is also not the cache is added for clarity and extra protection.

This test conveniently also tests cloning a repo which is not the
zuul_project, so add the original assertion back but modified for
that project.  This way both variants are now tested.

Change-Id: If22d8cfbe591afe4f5516da1eb3b0be98b2de874
This commit is contained in:
James E. Blair 2016-10-18 08:18:36 -07:00
parent bfd85fd509
commit 8cce42e329
4 changed files with 147 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
tests/base.py
View File

@ -262,6 +262,25 @@ class FakeChange(object):
"comment": "This is a comment"}
return event
def getRefUpdatedEvent(self):
path = os.path.join(self.upstream_root, self.project)
repo = git.Repo(path)
oldrev = repo.heads[self.branch].commit.hexsha
event = {
"type": "ref-updated",
"submitter": {
"name": "User Name",
},
"refUpdate": {
"oldRev": oldrev,
"newRev": self.patchsets[-1]['revision'],
"refName": self.branch,
"project": self.project,
}
}
return event
def addApproval(self, category, value, username='reviewer_john',
granted_on=None, message=''):
if not granted_on:

9
tests/fixtures/layout-cloner.yaml vendored
View File

@ -30,6 +30,13 @@ pipelines:
gerrit:
verified: -2
- name: post
manager: IndependentPipelineManager
trigger:
gerrit:
- event: ref-updated
ref: ^(?!refs/).*$
projects:
- name: org/project
check:
@ -42,6 +49,8 @@ projects:
- integration
gate:
- integration
post:
- postjob
- name: org/project2
check:

120
tests/test_cloner.py
View File

@ -108,11 +108,34 @@ class TestCloner(ZuulTestCase):
'be correct' % (project, number))
work = self.getWorkspaceRepos(projects)
upstream_repo_path = os.path.join(self.upstream_root, 'org/project1')
self.assertEquals(
# project1 is the zuul_project so the origin should be set to the
# zuul_url since that is the most up to date.
cache_repo_path = os.path.join(cache_root, 'org/project1')
self.assertNotEqual(
work['org/project1'].remotes.origin.url,
cache_repo_path,
'workspace repo origin should not be the cache'
)
zuul_url_repo_path = os.path.join(self.git_root, 'org/project1')
self.assertEqual(
work['org/project1'].remotes.origin.url,
zuul_url_repo_path,
'workspace repo origin should be the zuul url'
)
# project2 is not the zuul_project so the origin should be set
# to upstream since that is the best we can do
cache_repo_path = os.path.join(cache_root, 'org/project2')
self.assertNotEqual(
work['org/project2'].remotes.origin.url,
cache_repo_path,
'workspace repo origin should not be the cache'
)
upstream_repo_path = os.path.join(self.upstream_root, 'org/project2')
self.assertEqual(
work['org/project2'].remotes.origin.url,
upstream_repo_path,
'workspace repo origin should be upstream, not cache'
'workspace repo origin should be the upstream url'
)
self.worker.hold_jobs_in_build = False
@ -656,55 +679,76 @@ class TestCloner(ZuulTestCase):
self.waitUntilSettled()
def test_post_checkout(self):
project = "org/project"
path = os.path.join(self.upstream_root, project)
repo = git.Repo(path)
repo.head.reference = repo.heads['master']
commits = []
for i in range(0, 3):
commits.append(self.create_commit(project))
newRev = commits[1]
self.worker.hold_jobs_in_build = True
project = "org/project1"
A = self.fake_gerrit.addFakeChange(project, 'master', 'A')
event = A.getRefUpdatedEvent()
A.setMerged()
self.fake_gerrit.addEvent(event)
self.waitUntilSettled()
build = self.builds[0]
state = {'org/project1': build.parameters['ZUUL_COMMIT']}
build.release()
self.waitUntilSettled()
cloner = zuul.lib.cloner.Cloner(
git_base_url=self.upstream_root,
projects=[project],
workspace=self.workspace_root,
zuul_project='org/project',
zuul_branch=None,
zuul_ref='master',
zuul_project=build.parameters.get('ZUUL_PROJECT', None),
zuul_branch=build.parameters.get('ZUUL_BRANCH', None),
zuul_ref=build.parameters.get('ZUUL_REF', None),
zuul_newrev=build.parameters.get('ZUUL_NEWREV', None),
zuul_url=self.git_root,
zuul_newrev=newRev,
)
cloner.execute()
repos = self.getWorkspaceRepos([project])
cloned_sha = repos[project].rev_parse('HEAD').hexsha
self.assertEqual(newRev, cloned_sha)
work = self.getWorkspaceRepos([project])
self.assertEquals(state[project],
str(work[project].commit('HEAD')),
'Project %s commit for build %s should '
'be correct' % (project, 0))
shutil.rmtree(self.workspace_root)
def test_post_and_master_checkout(self):
project = "org/project1"
master_project = "org/project2"
path = os.path.join(self.upstream_root, project)
repo = git.Repo(path)
repo.head.reference = repo.heads['master']
commits = []
for i in range(0, 3):
commits.append(self.create_commit(project))
newRev = commits[1]
self.worker.hold_jobs_in_build = True
projects = ["org/project1", "org/project2"]
A = self.fake_gerrit.addFakeChange(projects[0], 'master', 'A')
event = A.getRefUpdatedEvent()
A.setMerged()
self.fake_gerrit.addEvent(event)
self.waitUntilSettled()
build = self.builds[0]
upstream = self.getUpstreamRepos(projects)
state = {'org/project1':
build.parameters['ZUUL_COMMIT'],
'org/project2':
str(upstream['org/project2'].commit('master')),
}
build.release()
self.waitUntilSettled()
cloner = zuul.lib.cloner.Cloner(
git_base_url=self.upstream_root,
projects=[project, master_project],
projects=projects,
workspace=self.workspace_root,
zuul_project='org/project1',
zuul_branch=None,
zuul_ref='master',
zuul_project=build.parameters.get('ZUUL_PROJECT', None),
zuul_branch=build.parameters.get('ZUUL_BRANCH', None),
zuul_ref=build.parameters.get('ZUUL_REF', None),
zuul_newrev=build.parameters.get('ZUUL_NEWREV', None),
zuul_url=self.git_root,
zuul_newrev=newRev
)
cloner.execute()
repos = self.getWorkspaceRepos([project, master_project])
cloned_sha = repos[project].rev_parse('HEAD').hexsha
self.assertEqual(newRev, cloned_sha)
self.assertEqual(
repos[master_project].rev_parse('HEAD').hexsha,
repos[master_project].rev_parse('master').hexsha)
work = self.getWorkspaceRepos(projects)
for project in projects:
self.assertEquals(state[project],
str(work[project].commit('HEAD')),
'Project %s commit for build %s should '
'be correct' % (project, 0))
shutil.rmtree(self.workspace_root)

49
zuul/lib/cloner.py
View File

@ -46,6 +46,8 @@ class Cloner(object):
self.zuul_branch = zuul_branch or ''
self.zuul_ref = zuul_ref or ''
self.zuul_url = zuul_url
self.zuul_project = zuul_project
self.project_branches = project_branches or {}
self.project_revisions = {}
@ -77,7 +79,18 @@ class Cloner(object):
def cloneUpstream(self, project, dest):
# Check for a cached git repo first
git_cache = '%s/%s' % (self.cache_dir, project)
git_upstream = '%s/%s' % (self.git_url, project)
# Then, if we are cloning the repo for the zuul_project, then
# set its origin to be the zuul merger, as it is guaranteed to
# be correct and up to date even if mirrors haven't updated
# yet. Otherwise, we can not be sure about the state of the
# project, so our best chance to get the most current state is
# by setting origin to the git_url.
if (self.zuul_url and project == self.zuul_project):
git_upstream = '%s/%s' % (self.zuul_url, project)
else:
git_upstream = '%s/%s' % (self.git_url, project)
repo_is_cloned = os.path.exists(os.path.join(dest, '.git'))
if (self.cache_dir and
os.path.exists(git_cache) and
@ -104,23 +117,35 @@ class Cloner(object):
return repo
def fetchFromZuul(self, repo, project, ref):
zuul_remote = '%s/%s' % (self.zuul_url, project)
def fetchRef(self, repo, project, ref):
# If we are fetching a zuul ref, the only place to get it is
# from the zuul merger (and it is guaranteed to be correct).
# Otherwise, the only way we can be certain that the ref
# (which, since it is not a zuul ref, is a branch or tag) is
# correct is in the case that it matches zuul_project. If
# neither of those two conditions are met, we are most likely
# to get the correct state from the git_url.
if (ref.startswith('refs/zuul') or
project == self.zuul_project):
remote = '%s/%s' % (self.zuul_url, project)
else:
remote = '%s/%s' % (self.git_url, project)
try:
repo.fetchFrom(zuul_remote, ref)
self.log.debug("Fetched ref %s from %s", ref, project)
repo.fetchFrom(remote, ref)
self.log.debug("Fetched ref %s from %s", ref, remote)
return True
except ValueError:
self.log.debug("Project %s in Zuul does not have ref %s",
project, ref)
self.log.debug("Repo %s does not have ref %s",
remote, ref)
return False
except GitCommandError as error:
# Bail out if fetch fails due to infrastructure reasons
if error.stderr.startswith('fatal: unable to access'):
raise
self.log.debug("Project %s in Zuul does not have ref %s",
project, ref)
self.log.debug("Repo %s does not have ref %s",
remote, ref)
return False
def prepareRepo(self, project, dest):
@ -192,7 +217,7 @@ class Cloner(object):
self.log.info("Attempting to check out revision %s for "
"project %s", indicated_revision, project)
try:
self.fetchFromZuul(repo, project, self.zuul_ref)
self.fetchRef(repo, project, self.zuul_ref)
commit = repo.checkout(indicated_revision)
except (ValueError, GitCommandError):
raise exceptions.RevNotFound(project, indicated_revision)
@ -201,10 +226,10 @@ class Cloner(object):
# If we have a non empty zuul_ref to use, use it. Otherwise we fall
# back to checking out the branch.
elif ((override_zuul_ref and
self.fetchFromZuul(repo, project, override_zuul_ref)) or
self.fetchRef(repo, project, override_zuul_ref)) or
(fallback_zuul_ref and
fallback_zuul_ref != override_zuul_ref and
self.fetchFromZuul(repo, project, fallback_zuul_ref))):
self.fetchRef(repo, project, fallback_zuul_ref))):
# Work around a bug in GitPython which can not parse FETCH_HEAD
gitcmd = git.Git(dest)
fetch_head = gitcmd.rev_parse('FETCH_HEAD')