diff --git a/fixtures_git/utils.py b/fixtures_git/_utils.py similarity index 96% rename from fixtures_git/utils.py rename to fixtures_git/_utils.py index aef5628..ba69b61 100644 --- a/fixtures_git/utils.py +++ b/fixtures_git/_utils.py @@ -16,7 +16,7 @@ import re -def _get_node_to_pick(node): +def get_node_to_pick(node): m = re.search(r'(.*)(\d+)$', node) if m: # get copy of a another change @@ -33,7 +33,7 @@ _VISITED = 1 _FINISHED = 2 -def _reverse_toposort(data): +def reverse_toposort(data): # convert to dict for linear lookup times when returning data = dict(data) @@ -63,7 +63,7 @@ def _reverse_toposort(data): visited[node] = _VISITED nodes_to_visit.append(node) # special case for cherry-picking changes - c_node = _get_node_to_pick(node) + c_node = get_node_to_pick(node) if c_node and c_node not in visited: nodes_to_visit.append(c_node) diff --git a/fixtures_git/gitfixture.py b/fixtures_git/gitfixture.py index 60bac0a..879cda3 100644 --- a/fixtures_git/gitfixture.py +++ b/fixtures_git/gitfixture.py @@ -13,15 +13,15 @@ # License for the specific language governing permissions and limitations # under the License. -import faker import os import shutil import tempfile +import faker import fixtures import git -from fixtures_git import utils +from . import gittree try: import urlparse as parse @@ -29,161 +29,6 @@ except ImportError: from urllib import parse -class GitTree(object): - """Helper class to build a git repository from a graph definition - - Helper class to build a git repository from a graph definition of - nodes and their parent nodes. A list of branches may be provided - where each element has two members corresponding to the name and the - target node it references. - - Supports unordered graphs, only requirement is that there is a commit - defined with no parents, which will become the root commit. - - Root commits can specified by an empty list as the second member: - - ('NodeA', []) - - Merge commits are specified by multiple nodes: - - ('NodeMerge', ['Node1', 'Node2']) - - - An '=' prefix can be used in front of one of the parents of a merge - commit to indicate that the merge commit's tree should be copied - directly from that parent ignoring any contributions from the other - parents' trees (like what git enables with the '-s ours' merge - strategy. - - This is used when needing to preserve history but not interested in - the changes from the other branches. One tool that makes extensive - usage of this is _git-upstream. - - E.g., the following will result in a merge commit 'C', with parents - 'P1' and 'P2', but will have the same tree as 'P1'. - - ('C', ['=P1', 'P2']) - - - The tree building code can handle a graph definition being out of - order but will fail to find certain circular dependencies and may - result in an infinite loop. - - Examples: - - [('A', []), ('B', ['A']), ('C', ['B'])] - [('A', []), ('C', ['B']), ('B', ['A'])] - - .. _git-upstream: https://pypi.python.org/pypi/git-upstream - """ - - def __init__(self, gitrepo, tree, branches): - self.graph = {} - self.gitrepo = gitrepo - self.repo = gitrepo.repo - self._build_git_tree(tree, branches) - - def _commit(self, node): - p_node = utils._get_node_to_pick(node) - if p_node: - self.repo.git.cherry_pick(self.graph[p_node], x=True) - else: - # standard commit - self.gitrepo.add_commits(1, ref="HEAD", - message_prefix="[%s]" % node) - - def _merge_commit(self, node, parents): - # merge commits - parent_nodes = [p.lstrip("=") for p in parents] - commits = [str(self.graph[p]) for p in parent_nodes[1:]] - - if any([p.startswith("=") for p in parents]): - # special merge commit using inverse of 'ours' by - # emptying the current index and then reading in any - # trees of the nodes prefixed with '=' - use = [str(self.graph[p.lstrip("=")]) - for p in parents if p.startswith("=")] - try: - self.repo.git.merge(*commits, s="ours", no_commit=True) - except git.exc.GitCommandError as exc: - if 'refusing to merge unrelated histories' in exc.stderr: - self.repo.git.merge(*commits, s="ours", no_commit=True, - allow_unrelated_histories=True) - else: - raise - self.repo.git.read_tree(empty=True) - self.repo.git.read_tree(empty=True) - self.repo.git.read_tree(*use, u=True, reset=True) - elif len(commits) < 2: - # standard merge - try: - self.repo.git.merge(*commits, no_commit=True) - except git.exc.GitCommandError as exc: - if 'refusing to merge unrelated histories' in exc.stderr: - self.repo.git.merge(*commits, no_commit=True, - allow_unrelated_histories=True) - else: - raise - else: - # multi-branch merge, git is not great at handling - # merging multiple orphaned branches - try: - self.repo.git.merge(*commits, s="ours", no_commit=True) - except git.exc.GitCommandError as exc: - if 'refusing to merge unrelated histories' in exc.stderr: - self.repo.git.merge(*commits, s="ours", no_commit=True, - allow_unrelated_histories=True) - else: - raise - self.repo.git.read_tree(empty=True) - self.repo.git.read_tree("HEAD", *commits) - self.repo.git.checkout("--", ".") - self.repo.git.commit(m="[%s] Merging %s into %s" % - (node, ",".join(parent_nodes[1:]), - parent_nodes[0])) - self.repo.git.clean(f=True, d=True, x=True) - - def _build_git_tree(self, graph_def, branches=[]): - - # require that graphs must have at least 1 node with no - # parents, which is a root commit in git - if not any([True for _, parents in graph_def if not parents]): - assert("No root commit defined in test graph") - - for node, parents in utils._reverse_toposort(graph_def): - if not parents: - # root commit - self.repo.git.symbolic_ref("HEAD", "refs/heads/%s" % node) - self.repo.git.rm(".", r=True, cached=True, - with_exceptions=False) - self.repo.git.clean(f=True, d=True, x=True) - self.gitrepo.add_commits(1, ref="HEAD", - message_prefix="[%s]" % node) - # only explicitly listed branches should exist afterwards - self.repo.git.checkout(self.repo.commit()) - self.repo.git.branch(node, D=True) - - else: - # checkout the dependent node - self.repo.git.checkout(self.graph[parents[0].lstrip('=')]) - if len(parents) > 1: - # merge commits - self._merge_commit(node, parents) - else: - self._commit(node) - self.graph[node] = self.repo.commit() - - for name, node in branches: - self.repo.git.branch(name, str(self.graph[node]), f=True) - - # return to master - self.repo.git.checkout("master") - - def commits_from_nodes(self, nodes=[]): - - return [self.graph[n] for n in nodes] - - class GitFixture(fixtures.Fixture): """Create a git repo in which to operate. @@ -243,7 +88,7 @@ class GitFixture(fixtures.Fixture): self.repo.git.commit(m="Initialize empty repo", allow_empty=True) if self._graph: - self.gittree = GitTree(self, self._graph, self._branches) + self.gittree = gittree.GitTree(self, self._graph, self._branches) def _create_file(self): contents = "\n\n".join(self._faker.paragraphs(3)) @@ -276,9 +121,12 @@ class GitFixture(fixtures.Fixture): message = message + "\n\nChange-Id: %s" % change_id self.repo.git.commit(m=message) - def add_commits(self, num=None, ref="HEAD", change_ids=[], + def add_commits(self, num=None, ref="HEAD", change_ids=None, message_prefix=None): """Create the given number of commits using generated files""" + if change_ids is None: + change_ids = [] + if ref != "HEAD": self.repo.git.checkout(ref) diff --git a/fixtures_git/gittree.py b/fixtures_git/gittree.py new file mode 100644 index 0000000..221633c --- /dev/null +++ b/fixtures_git/gittree.py @@ -0,0 +1,176 @@ +# Copyright (c) 2018 Hewlett Packard Enterprise Development Company LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +import git + +from fixtures_git import _utils + + +class GitTree(object): + """Helper class to build a git repository from a graph definition + + Helper class to build a git repository from a graph definition of + nodes and their parent nodes. A list of branches may be provided + where each element has two members corresponding to the name and the + target node it references. + + Supports unordered graphs, only requirement is that there is a commit + defined with no parents, which will become the root commit. + + Root commits can specified by an empty list as the second member: + + ('NodeA', []) + + Merge commits are specified by multiple nodes: + + ('NodeMerge', ['Node1', 'Node2']) + + + An '=' prefix can be used in front of one of the parents of a merge + commit to indicate that the merge commit's tree should be copied + directly from that parent ignoring any contributions from the other + parents' trees (like what git enables with the '-s ours' merge + strategy. + + This is used when needing to preserve history but not interested in + the changes from the other branches. One tool that makes extensive + usage of this is _git-upstream. + + E.g., the following will result in a merge commit 'C', with parents + 'P1' and 'P2', but will have the same tree as 'P1'. + + ('C', ['=P1', 'P2']) + + + The tree building code can handle a graph definition being out of + order but will fail to find certain circular dependencies and may + result in an infinite loop. + + Examples: + + [('A', []), ('B', ['A']), ('C', ['B'])] + [('A', []), ('C', ['B']), ('B', ['A'])] + + .. _git-upstream: https://pypi.python.org/pypi/git-upstream + """ + + def __init__(self, gitrepo, tree, branches): + self.graph = {} + self.gitrepo = gitrepo + self.repo = gitrepo.repo + self._build_git_tree(tree, branches) + + def _commit(self, node): + p_node = _utils.get_node_to_pick(node) + if p_node: + self.repo.git.cherry_pick(self.graph[p_node], x=True) + else: + # standard commit + self.gitrepo.add_commits(1, ref="HEAD", + message_prefix="[%s]" % node) + + def _merge_commit(self, node, parents): + # merge commits + parent_nodes = [p.lstrip("=") for p in parents] + commits = [str(self.graph[p]) for p in parent_nodes[1:]] + + if any([p.startswith("=") for p in parents]): + # special merge commit using inverse of 'ours' by + # emptying the current index and then reading in any + # trees of the nodes prefixed with '=' + use = [str(self.graph[p.lstrip("=")]) + for p in parents if p.startswith("=")] + try: + self.repo.git.merge(*commits, s="ours", no_commit=True) + except git.exc.GitCommandError as exc: + if 'refusing to merge unrelated histories' in exc.stderr: + self.repo.git.merge(*commits, s="ours", no_commit=True, + allow_unrelated_histories=True) + else: + raise + self.repo.git.read_tree(empty=True) + self.repo.git.read_tree(empty=True) + self.repo.git.read_tree(*use, u=True, reset=True) + elif len(commits) < 2: + # standard merge + try: + self.repo.git.merge(*commits, no_commit=True) + except git.exc.GitCommandError as exc: + if 'refusing to merge unrelated histories' in exc.stderr: + self.repo.git.merge(*commits, no_commit=True, + allow_unrelated_histories=True) + else: + raise + else: + # multi-branch merge, git is not great at handling + # merging multiple orphaned branches + try: + self.repo.git.merge(*commits, s="ours", no_commit=True) + except git.exc.GitCommandError as exc: + if 'refusing to merge unrelated histories' in exc.stderr: + self.repo.git.merge(*commits, s="ours", no_commit=True, + allow_unrelated_histories=True) + else: + raise + self.repo.git.read_tree(empty=True) + self.repo.git.read_tree("HEAD", *commits) + self.repo.git.checkout("--", ".") + self.repo.git.commit(m="[%s] Merging %s into %s" % + (node, ",".join(parent_nodes[1:]), + parent_nodes[0])) + self.repo.git.clean(f=True, d=True, x=True) + + def _build_git_tree(self, graph_def, branches=None): + + if branches is None: + branches = [] + + # require that graphs must have at least 1 node with no + # parents, which is a root commit in git + if not any([True for _, parents in graph_def if not parents]): + assert "No root commit defined in test graph" + + for node, parents in _utils.reverse_toposort(graph_def): + if not parents: + # root commit + self.repo.git.symbolic_ref("HEAD", "refs/heads/%s" % node) + self.repo.git.rm(".", r=True, cached=True, + with_exceptions=False) + self.repo.git.clean(f=True, d=True, x=True) + self.gitrepo.add_commits(1, ref="HEAD", + message_prefix="[%s]" % node) + # only explicitly listed branches should exist afterwards + self.repo.git.checkout(self.repo.commit()) + self.repo.git.branch(node, D=True) + + else: + # checkout the dependent node + self.repo.git.checkout(self.graph[parents[0].lstrip('=')]) + if len(parents) > 1: + # merge commits + self._merge_commit(node, parents) + else: + self._commit(node) + self.graph[node] = self.repo.commit() + + for name, node in branches: + self.repo.git.branch(name, str(self.graph[node]), f=True) + + # return to master + self.repo.git.checkout("master") + + def commits_from_nodes(self, nodes): + + return [self.graph[n] for n in nodes] diff --git a/fixtures_git/reposet.py b/fixtures_git/reposet.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/utils/test_toposort.py b/tests/unit/utils/test_toposort.py index d559714..cdddef5 100644 --- a/tests/unit/utils/test_toposort.py +++ b/tests/unit/utils/test_toposort.py @@ -16,7 +16,7 @@ import testtools -from fixtures_git import utils +from fixtures_git import _utils from tests import base @@ -30,7 +30,7 @@ class TestResolve(testtools.TestCase): ] self.assertEqual( nodes, - list(utils._reverse_toposort(nodes)), + list(_utils.reverse_toposort(nodes)), ) def test_unordered(self): @@ -41,7 +41,7 @@ class TestResolve(testtools.TestCase): ] self.assertEqual( [('A', []), ('B', ['A']), ('C', ['B'])], - list(utils._reverse_toposort(nodes)) + list(_utils.reverse_toposort(nodes)) ) def test_merge(self): @@ -52,7 +52,7 @@ class TestResolve(testtools.TestCase): ('E', ['D', 'C']), ('D', ['A']), ] - sorted = list(utils._reverse_toposort(nodes)) + sorted = list(_utils.reverse_toposort(nodes)) self.assertThat( (nodes[2], nodes[0], nodes[1], nodes[3]), base.IsOrderedSubsetOf(sorted) @@ -72,7 +72,7 @@ class TestResolve(testtools.TestCase): ('G', ['F', 'C']), ('F', ['A']), ] - sorted = list(utils._reverse_toposort(nodes)) + sorted = list(_utils.reverse_toposort(nodes)) # A -> B -> C -> E self.assertThat( (nodes[2], nodes[0], nodes[1], nodes[3]), @@ -101,7 +101,7 @@ class TestResolve(testtools.TestCase): ('A', []), # root commit ('D', ['B', 'C']), ] - sorted = list(utils._reverse_toposort(nodes)) + sorted = list(_utils.reverse_toposort(nodes)) # assert partial ordering because nodes A & B may come # before or after node C. Just make sure that node D # is defined after them.