Initial fixture for git

Provide fixture object that can be used to build isolated test envs
when working with git and developing git tooling that uses the git
application.

Some basic tests on the topological sorting to ensure nodes are
returned in an expected order without requiring the code to enforce the
order.

Some simple acceptance tests ensure that the fixture will accept various
simple graphs and handle them correctly.

Change-Id: I5b39f8d52beca848adb20d646c26d91c4b977b92
This commit is contained in:
Darragh Bailey 2018-03-10 11:17:45 +00:00 committed by Darragh Bailey
parent 3cb59e0520
commit 248d079cc0
16 changed files with 740 additions and 3 deletions

1
.gitignore vendored
View File

@ -46,6 +46,7 @@ coverage.xml
*.cover
.hypothesis/
.pytest_cache/
.stestr/
# Translations
*.mo

3
.stestr.conf Normal file
View File

@ -0,0 +1,3 @@
[DEFAULT]
test_path=tests
top_dir=./

2
doc/source/git.rst Normal file
View File

@ -0,0 +1,2 @@
.. automodule:: fixtures_git.git
:members:

20
fixtures_git/__init__.py Normal file
View File

@ -0,0 +1,20 @@
# Copyright (c) 2018 Hewlett Packard Enterprise Development Company LP
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from fixtures_git.gitfixture import GitFixture
__all__ = [
'GitFixture'
]

286
fixtures_git/gitfixture.py Normal file
View File

@ -0,0 +1,286 @@
# Copyright (c) 2013-2016 Hewlett-Packard Development Company, L.P.
# Copyright (c) 2018 Hewlett Packard Enterprise Development Company LP
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import faker
import os
import shutil
import tempfile
import fixtures
import git
from fixtures_git import utils
class GitTree(object):
"""Helper class to build a git repository from a graph definition
Helper class to build a git repository from a graph definition of
nodes and their parent nodes. A list of branches may be provided
where each element has two members corresponding to the name and the
target node it references.
Supports unordered graphs, only requirement is that there is a commit
defined with no parents, which will become the root commit.
Root commits can specified by an empty list as the second member:
('NodeA', [])
Merge commits are specified by multiple nodes:
('NodeMerge', ['Node1', 'Node2'])
An '=' prefix can be used in front of one of the parents of a merge
commit to indicate that the merge commit's tree should be copied
directly from that parent ignoring any contributions from the other
parents' trees (like what git enables with the '-s ours' merge
strategy.
This is used when needing to preserve history but not interested in
the changes from the other branches. One tool that makes extensive
usage of this is _git-upstream.
E.g., the following will result in a merge commit 'C', with parents
'P1' and 'P2', but will have the same tree as 'P1'.
('C', ['=P1', 'P2'])
The tree building code can handle a graph definition being out of
order but will fail to find certain circular dependencies and may
result in an infinite loop.
Examples:
[('A', []), ('B', ['A']), ('C', ['B'])]
[('A', []), ('C', ['B']), ('B', ['A'])]
.. _git-upstream: https://pypi.python.org/pypi/git-upstream
"""
def __init__(self, gitrepo, tree, branches):
self.graph = {}
self.gitrepo = gitrepo
self.repo = gitrepo.repo
self._build_git_tree(tree, branches)
def _commit(self, node):
p_node = utils._get_node_to_pick(node)
if p_node:
self.repo.git.cherry_pick(self.graph[p_node], x=True)
else:
# standard commit
self.gitrepo.add_commits(1, ref="HEAD",
message_prefix="[%s]" % node)
def _merge_commit(self, node, parents):
# merge commits
parent_nodes = [p.lstrip("=") for p in parents]
commits = [str(self.graph[p]) for p in parent_nodes[1:]]
if any([p.startswith("=") for p in parents]):
# special merge commit using inverse of 'ours' by
# emptying the current index and then reading in any
# trees of the nodes prefixed with '='
use = [str(self.graph[p.lstrip("=")])
for p in parents if p.startswith("=")]
try:
self.repo.git.merge(*commits, s="ours", no_commit=True)
except git.exc.GitCommandError as exc:
if 'refusing to merge unrelated histories' in exc.stderr:
self.repo.git.merge(*commits, s="ours", no_commit=True,
allow_unrelated_histories=True)
else:
raise
self.repo.git.read_tree(empty=True)
self.repo.git.read_tree(empty=True)
self.repo.git.read_tree(*use, u=True, reset=True)
elif len(commits) < 2:
# standard merge
try:
self.repo.git.merge(*commits, no_commit=True)
except git.exc.GitCommandError as exc:
if 'refusing to merge unrelated histories' in exc.stderr:
self.repo.git.merge(*commits, no_commit=True,
allow_unrelated_histories=True)
else:
raise
else:
# multi-branch merge, git is not great at handling
# merging multiple orphaned branches
try:
self.repo.git.merge(*commits, s="ours", no_commit=True)
except git.exc.GitCommandError as exc:
if 'refusing to merge unrelated histories' in exc.stderr:
self.repo.git.merge(*commits, s="ours", no_commit=True,
allow_unrelated_histories=True)
else:
raise
self.repo.git.read_tree(empty=True)
self.repo.git.read_tree("HEAD", *commits)
self.repo.git.checkout("--", ".")
self.repo.git.commit(m="[%s] Merging %s into %s" %
(node, ",".join(parent_nodes[1:]),
parent_nodes[0]))
self.repo.git.clean(f=True, d=True, x=True)
def _build_git_tree(self, graph_def, branches=[]):
# require that graphs must have at least 1 node with no
# parents, which is a root commit in git
if not any([True for _, parents in graph_def if not parents]):
assert("No root commit defined in test graph")
for node, parents in utils._reverse_toposort(graph_def):
if not parents:
# root commit
self.repo.git.symbolic_ref("HEAD", "refs/heads/%s" % node)
self.repo.git.rm(".", r=True, cached=True,
with_exceptions=False)
self.repo.git.clean(f=True, d=True, x=True)
self.gitrepo.add_commits(1, ref="HEAD",
message_prefix="[%s]" % node)
# only explicitly listed branches should exist afterwards
self.repo.git.checkout(self.repo.commit())
self.repo.git.branch(node, D=True)
else:
# checkout the dependent node
self.repo.git.checkout(self.graph[parents[0].lstrip('=')])
if len(parents) > 1:
# merge commits
self._merge_commit(node, parents)
else:
self._commit(node)
self.graph[node] = self.repo.commit()
for name, node in branches:
self.repo.git.branch(name, str(self.graph[node]), f=True)
# return to master
self.repo.git.checkout("master")
def commits_from_nodes(self, nodes=[]):
return [self.graph[n] for n in nodes]
class GitFixture(fixtures.Fixture):
"""Create a git repo in which to operate.
By default creates an empty git repository under a temporary
directory and deletes it after use.
It accepts options to automatically define a git repository
layout based on list of commits setting the given branches to
the relevant node once built.
:ivar graph: Iterable describing the tree of git commits to create.
:ivar branches: Dict of node to branch names to set once finished.
:ivar path: Custom path to use, otherwise will create a temporary
directory to use and set the 'path' attribute to it.
:ivar user: Dict describing a user to use for commits, defaults
to 'Example User <user@example.com>',
:ivar clean_on_exit: Control whether to delete the tempoary path
once complete, defaults to 'True', but is ignored if 'path'
is provided.
"""
def __init__(self, graph=None, branches=None, path=None, user=None,
clean_on_exit=True):
# set attributes for use
self.path = path
self.gittree = None
self.repo = None
# internal attributes
self._graph = graph or []
self._branches = branches or []
self._user = {
'name': 'Example User',
'email': 'user@example.com',
}
self._user.update(user or {})
self._clean_on_exit = clean_on_exit
# for text generation
self._faker = faker.Faker()
def _setUp(self):
self._file_list = set()
if not self.path:
tempdir = self.useFixture(fixtures.TempDir())
self.path = os.path.join(tempdir.path, 'git')
if self._clean_on_exit is True:
self.addCleanup(shutil.rmtree, tempdir.path)
os.mkdir(self.path)
g = git.Git(self.path)
g.init()
self.repo = git.Repo(self.path)
self.repo.git.config('user.email', self._user['email'])
self.repo.git.config('user.name', self._user['name'])
self.repo.git.commit(m="Initialize empty repo", allow_empty=True)
if self._graph:
self.gittree = GitTree(self, self._graph, self._branches)
def _create_file(self):
contents = "\n\n".join(self._faker.paragraphs(3))
# always want to ensure the files added to the repo are unique no
# matter which branch they are added to, as otherwise there may
# be conflicts caused by replaying local changes and performing
# merges
while True:
tmpfile = tempfile.NamedTemporaryFile(
dir=self.repo.working_dir, delete=False)
if tmpfile.name not in self._file_list:
self._file_list.add(tmpfile.name)
break
# case where same filename in use on a different branch
tmpfile.close()
os.remove(tmpfile.name)
tmpfile.write(contents.encode('utf-8'))
tmpfile.close()
return tmpfile.name
def _create_file_commit(self, change_id=None, message_prefix=None):
filename = self._create_file()
self.repo.git.add(filename)
message = "Adding %s" % os.path.basename(filename)
if message_prefix:
message = "%s %s" % (message_prefix, message)
if change_id:
message = message + "\n\nChange-Id: %s" % change_id
self.repo.git.commit(m=message)
def add_commits(self, num=None, ref="HEAD", change_ids=[],
message_prefix=None):
"""Create the given number of commits using generated files"""
if ref != "HEAD":
self.repo.git.checkout(ref)
if num is None:
num = max(1, len(change_ids))
ids = list(change_ids) + [None] * (num - len(change_ids))
for x in range(num):
self._create_file_commit(
change_id=ids[x], message_prefix=message_prefix)

81
fixtures_git/utils.py Normal file
View File

@ -0,0 +1,81 @@
# Copyright (c) 2013-2016 Hewlett-Packard Development Company, L.P.
# Copyright (c) 2018 Hewlett Packard Enterprise Development Company LP
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import re
def _get_node_to_pick(node):
m = re.search(r'(.*)(\d+)$', node)
if m:
# get copy of a another change
node_number = int(m.group(2)) - 1
node_name = m.group(1)
if node_number > 0:
node_name += str(node_number)
return node_name
return None
_NOT_VISITED = 0
_VISITED = 1
_FINISHED = 2
def _reverse_toposort(data):
# convert to dict for linear lookup times when returning
data = dict(data)
# keep track of nodes visited and processed
# by checking if a child has been visited before but not processed you
# can detect a back edge and abort since the graph is not acyclic
visited = dict()
# DFS algorithm with customization to handle use of '=' notation for merge
# commits and also the additional dependency for cherry-picking
nodes_to_visit = []
for i in data.keys():
if i not in visited:
nodes_to_visit.append(i)
while nodes_to_visit:
node = nodes_to_visit.pop()
if visited.get(node) is _VISITED:
# already visited so just return it with it's deps
yield (node, data[node])
visited[node] = _FINISHED
continue
elif visited.get(node) is _FINISHED:
continue
visited[node] = _VISITED
nodes_to_visit.append(node)
# special case for cherry-picking changes
c_node = _get_node_to_pick(node)
if c_node and c_node not in visited:
nodes_to_visit.append(c_node)
for d in data[node]:
r_d = d.strip('=')
if r_d not in visited:
nodes_to_visit.append(r_d)
else:
# if we've already visited a dep but not processed it,
# then we have a back edge of some kind
if visited[r_d] is _VISITED:
message = ("Graph is not acyclic: %s is a dependency "
"of %s, but has been visited without being "
"processed before it." % (r_d, node))
raise RuntimeError(message)

3
requirements.txt Normal file
View File

@ -0,0 +1,3 @@
faker>=0.8.16
fixtures>=3.0.0
GitPython>=1.0.1

View File

@ -1 +1,5 @@
sphinx
flake8
hacking
sphinx>=1.6.5
stestr>=1.0.0
testtools>=2.2.0

0
tests/__init__.py Normal file
View File

View File

@ -0,0 +1,28 @@
#
# Copyright (c) 2018 Hewlett Packard Enterprise Development Company LP
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import logging
import fixtures
import testtools
class BaseTestCase(testtools.TestCase):
def setUp(self):
super(BaseTestCase, self).setUp()
self.logger = self.useFixture(fixtures.FakeLogger(level=logging.DEBUG))

View File

@ -0,0 +1,129 @@
# Copyright (c) 2018 Hewlett Packard Enterprise Development Company LP
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from testtools import matchers
from fixtures_git.gitfixture import GitFixture
from tests import acceptance
class TestGitFixture(acceptance.BaseTestCase):
def test_basic(self):
gitfixture = self.useFixture(
GitFixture(
[['A', []],
['B', ['A']],
['C', ['B']],
],
[['master', 'C']],
),
)
nodes = gitfixture.gittree.graph
self.assertEqual(len(list(gitfixture.repo.iter_commits())), 3)
self.assertTrue(gitfixture.repo.is_ancestor(nodes['A'], nodes['C']))
self.assertTrue(gitfixture.repo.is_ancestor(nodes['A'], nodes['B']))
self.assertEqual(gitfixture.repo.commit('master'), nodes['C'])
def test_merge(self):
gitfixture = self.useFixture(
GitFixture(
[['A', []],
['B', ['A']],
['C', ['B']],
['D', ['A']],
['E', ['D']],
['F', ['C', 'E']],
]
),
)
nodes = gitfixture.gittree.graph
self.assertTrue(gitfixture.repo.is_ancestor(nodes['B'], nodes['F']))
self.assertTrue(gitfixture.repo.is_ancestor(nodes['D'], nodes['F']))
self.assertEqual(len(gitfixture.repo.commit(nodes['F']).parents), 2)
self.assertEqual(gitfixture.repo.merge_base(nodes['E'], nodes['C']),
[nodes['A']])
node_f_files = gitfixture.repo.git.ls_files(
with_tree=nodes['F']).split('\n')
node_e_files = gitfixture.repo.git.ls_files(
with_tree=nodes['E']).split('\n')
node_c_files = gitfixture.repo.git.ls_files(
with_tree=nodes['C']).split('\n')
self.assertThat(
sorted(node_f_files),
matchers.NotEquals(sorted(node_c_files))
)
self.assertThat(
sorted(node_f_files),
matchers.Equals(sorted(set(node_c_files + node_e_files)))
)
def test_merge_and_replace(self):
gitfixture = self.useFixture(
GitFixture(
[['A', []],
['B', ['A']],
['C', ['B']],
['D', ['A']],
['E', ['D']],
['F', ['=C', 'E']],
]
),
)
nodes = gitfixture.gittree.graph
node_f_files = gitfixture.repo.git.ls_files(
with_tree=nodes['F']).split('\n')
node_c_files = gitfixture.repo.git.ls_files(
with_tree=nodes['C']).split('\n')
self.assertThat(
sorted(node_f_files),
matchers.Equals(sorted(node_c_files))
)
def test_unrelated_history(self):
gitfixture = self.useFixture(
GitFixture(
[['A', []],
['B', ['A']],
['C', ['B']],
['D', []],
['E', ['D']],
['F', ['C', 'E']],
]
),
)
nodes = gitfixture.gittree.graph
self.assertFalse(gitfixture.repo.is_ancestor(nodes['A'], nodes['D']))
self.assertFalse(gitfixture.repo.is_ancestor(nodes['D'], nodes['A']))
self.assertEqual(gitfixture.repo.merge_base(nodes['C'], nodes['E']),
[])
node_f_files = gitfixture.repo.git.ls_files(
with_tree=nodes['F']).split('\n')
node_e_files = gitfixture.repo.git.ls_files(
with_tree=nodes['E']).split('\n')
node_c_files = gitfixture.repo.git.ls_files(
with_tree=nodes['C']).split('\n')
self.assertThat(
sorted(node_f_files),
matchers.Equals(sorted(set(node_c_files + node_e_files))
)
)

45
tests/base.py Normal file
View File

@ -0,0 +1,45 @@
# Copyright (c) 2018 Hewlett Packard Enterprise Development LP
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
class IsOrderedSubsetOfMismatch(object):
def __init__(self, subset, set):
self.subset = list(subset)
self.set = list(set)
def describe(self):
return "set %r is not an ordered subset of %r" % (
self.subset, self.set)
def get_details(self):
return {}
class IsOrderedSubsetOf(object):
"""Matches if the actual matches the order of iterable."""
def __init__(self, iterable):
self.iterable = iterable
def __str__(self):
return 'IsOrderedSubsetOf(%s)' % self.iterable
def match(self, actual):
iterable = iter(self.iterable)
if all(item in iterable for item in actual):
return None
else:
return IsOrderedSubsetOfMismatch(actual, self.iterable)

0
tests/unit/__init__.py Normal file
View File

View File

View File

@ -0,0 +1,115 @@
# Copyright (c) 2018 Hewlett Packard Enterprise Development LP
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import testtools
from fixtures_git import utils
from tests import base
class TestResolve(testtools.TestCase):
def test_ordered(self):
nodes = [
('A', []),
('B', ['A']),
('C', ['B']),
]
self.assertEqual(
nodes,
list(utils._reverse_toposort(nodes)),
)
def test_unordered(self):
nodes = [
('B', ['A']),
('C', ['B']),
('A', []),
]
self.assertEqual(
[('A', []), ('B', ['A']), ('C', ['B'])],
list(utils._reverse_toposort(nodes))
)
def test_merge(self):
nodes = [
('B', ['A']),
('C', ['B']),
('A', []),
('E', ['D', 'C']),
('D', ['A']),
]
sorted = list(utils._reverse_toposort(nodes))
self.assertThat(
(nodes[2], nodes[0], nodes[1], nodes[3]),
base.IsOrderedSubsetOf(sorted)
)
self.assertThat(
(nodes[2], nodes[4], nodes[3]),
base.IsOrderedSubsetOf(sorted)
)
def test_multiple_merges(self):
nodes = [
('B', ['A']),
('C', ['B']),
('A', []),
('E', ['D', 'C']),
('D', ['A']),
('G', ['F', 'C']),
('F', ['A']),
]
sorted = list(utils._reverse_toposort(nodes))
# A -> B -> C -> E
self.assertThat(
(nodes[2], nodes[0], nodes[1], nodes[3]),
base.IsOrderedSubsetOf(sorted)
)
# A -> D -> E
self.assertThat(
(nodes[2], nodes[4], nodes[3]),
base.IsOrderedSubsetOf(sorted)
)
# A -> F -> G
self.assertThat(
(nodes[2], nodes[6], nodes[5]),
base.IsOrderedSubsetOf(sorted)
)
# A -> B -> C -> G
self.assertThat(
(nodes[2], nodes[0], nodes[1], nodes[5]),
base.IsOrderedSubsetOf(sorted)
)
def test_merge_multiple_roots(self):
nodes = [
('B', ['A']),
('C', []), # root commit
('A', []), # root commit
('D', ['B', 'C']),
]
sorted = list(utils._reverse_toposort(nodes))
# assert partial ordering because nodes A & B may come
# before or after node C. Just make sure that node D
# is defined after them.
self.assertThat(
(nodes[2], nodes[0], nodes[3]),
base.IsOrderedSubsetOf(sorted),
)
self.assertThat(
(nodes[1], nodes[3]),
base.IsOrderedSubsetOf(sorted),
)

24
tox.ini
View File

@ -6,10 +6,25 @@ skip_missing_interpreters = True
[testenv]
usedevelop = True
setenv =
LANG=en_US.UTF-8
PYTHONDONTWRITEBYTECODE=1
SUBUNIT_FORMATTER=tee testr_subunit_log
VIRTUAL_ENV={envdir}
install_command = pip install -U {opts} {packages}
deps = -r{toxinidir}/test-requirements.txt
deps = -r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
commands =
python setup.py test
stestr run --slowest {posargs}
[testenv:pep8]
commands = flake8
[testenv:cover]
commands =
python setup.py stestr --coverage --coverage-package-name=fixtures_git
coverage report
[testenv:docs]
commands =
@ -17,3 +32,8 @@ commands =
[testenv:venv]
commands = {posargs}
[flake8]
ignore=H236,H40
show-source = True
exclude = .venv,.tox,dist,doc,build,*.egg