diff --git a/packetary/controllers/repository.py b/packetary/controllers/repository.py index d1a615f..539df7e 100644 --- a/packetary/controllers/repository.py +++ b/packetary/controllers/repository.py @@ -130,7 +130,9 @@ class RepositoryController(object): :param package_files: the list of paths of packages :return : the new repository """ - repo = self.driver.create_repository(repository_data, self.arch) + repo = self.driver.create_repository( + self.context.connection, repository_data, self.arch + ) packages = set() with self.context.async_section() as section: for url in package_files: diff --git a/packetary/drivers/base.py b/packetary/drivers/base.py index 0a5c421..dd7c2d9 100644 --- a/packetary/drivers/base.py +++ b/packetary/drivers/base.py @@ -77,9 +77,10 @@ class RepositoryDriverBase(object): """ @abc.abstractmethod - def create_repository(self, repository_data, arch): + def create_repository(self, connection, repository_data, arch): """Create new repository. + :param connection: the connection manager instance :param repository_data: repository input data :param arch: the repository`s architecture :return: new repository object diff --git a/packetary/drivers/deb_driver.py b/packetary/drivers/deb_driver.py index 2a9601c..0629f2e 100644 --- a/packetary/drivers/deb_driver.py +++ b/packetary/drivers/deb_driver.py @@ -206,7 +206,7 @@ class DebRepositoryDriver(RepositoryDriverBase): self._create_repository_structure(new_repo) return new_repo - def create_repository(self, repository_data, arch): + def create_repository(self, connection, repository_data, arch): url = utils.normalize_repository_url(repository_data['uri']) suite = repository_data['suite'] component = repository_data.get('section') diff --git a/packetary/drivers/rpm_driver.py b/packetary/drivers/rpm_driver.py index e6dd9ea..d98eeaa 100644 --- a/packetary/drivers/rpm_driver.py +++ b/packetary/drivers/rpm_driver.py @@ -160,7 +160,7 @@ class RpmRepositoryDriver(RepositoryDriverBase): def add_packages(self, connection, repository, packages): groupstree = self._load_groups(connection, repository) - self._rebuild_repository(repository, packages, groupstree) + self._rebuild_repository(connection, repository, packages, groupstree) def fork_repository(self, connection, repository, destination, source=False, locale=False): @@ -170,10 +170,10 @@ class RpmRepositoryDriver(RepositoryDriverBase): new_repo.url = utils.normalize_repository_url(destination) utils.ensure_dir_exist(destination) groupstree = self._load_groups(connection, repository) - self._rebuild_repository(new_repo, None, groupstree) + self._rebuild_repository(connection, new_repo, set(), groupstree) return new_repo - def create_repository(self, repository_data, arch): + def create_repository(self, connection, repository_data, arch): repository = Repository( name=repository_data['name'], url=utils.normalize_repository_url(repository_data["uri"]), @@ -182,7 +182,7 @@ class RpmRepositoryDriver(RepositoryDriverBase): origin=repository_data.get('origin') ) utils.ensure_dir_exist(utils.get_path_from_url(repository.url)) - self._rebuild_repository(repository, None, None) + self._rebuild_repository(connection, repository, None, None) return repository def load_package_from_file(self, repository, filepath): @@ -211,24 +211,30 @@ class RpmRepositoryDriver(RepositoryDriverBase): def get_relative_path(self, repository, filename): return "packages/" + filename - def _rebuild_repository(self, repository, packages, groupstree=None): - basepath = utils.get_path_from_url(repository.url) + def _rebuild_repository(self, conn, repo, packages, groupstree=None): + basepath = utils.get_path_from_url(repo.url) self.logger.info("rebuild repository in %s", basepath) md_config = createrepo.MetaDataConfig() - update = packages is not None and \ - os.path.exists(os.path.join(basepath, md_config.finaldir)) - + mdfile_path = os.path.join( + basepath, md_config.finaldir, md_config.repomdfile + ) + update = packages is not None and os.path.exists(mdfile_path) groupsfile = None + if groupstree is None and update: + # The createrepo lose the groups information on update + # to prevent this set group info manually + groupstree = self._load_groups(conn, repo) + if groupstree is not None: - with tempfile.NamedTemporaryFile(delete=False) as tmp: - groupstree.write(tmp) - groupsfile = tmp.name + groupsfile = os.path.join(tempfile.gettempdir(), "groups.xml") + with open(groupsfile, "w") as fd: + groupstree.write(fd) try: md_config.workers = multiprocessing.cpu_count() md_config.directory = str(basepath) md_config.groupfile = groupsfile md_config.update = update - if packages is None: + if not packages: # only generate meta-files, without packages info md_config.excludes = ["*"] diff --git a/packetary/objects/packages_forest.py b/packetary/objects/packages_forest.py index c8051e0..19a9fdc 100644 --- a/packetary/objects/packages_forest.py +++ b/packetary/objects/packages_forest.py @@ -54,28 +54,29 @@ class PackagesForest(object): resolved = set() unresolved = set() - stack = [requirements] + stack = [(None, requirements)] if include_mandatory: for tree in self.trees: for mandatory in tree.mandatory_packages: resolved.add(mandatory) - stack.append(mandatory.requires) + stack.append((mandatory, mandatory.requires)) while stack: - requirements = stack.pop() + pkg, requirements = stack.pop() for required in requirements: for rel in required: if rel not in unresolved: candidate = self.find(rel) if candidate is not None: if candidate not in resolved: - stack.append(candidate.requires) + stack.append((candidate, candidate.requires)) resolved.add(candidate) break else: unresolved.add(required) - logger.warning("Unresolved relation: %s", required) + logger.warning("Unresolved relation: %s from %s", + required, pkg and pkg.name) return resolved def find(self, relation): diff --git a/packetary/tests/stubs/helpers.py b/packetary/tests/stubs/helpers.py index 79a27b0..317e923 100644 --- a/packetary/tests/stubs/helpers.py +++ b/packetary/tests/stubs/helpers.py @@ -22,6 +22,11 @@ import mock import six +class HTTPError(Exception): + def __init__(self, code): + self.code = code + + class CallbacksAdapter(mock.MagicMock): """Helper to return data through callback.""" diff --git a/packetary/tests/test_deb_driver.py b/packetary/tests/test_deb_driver.py index b6be505..e58f8f5 100644 --- a/packetary/tests/test_deb_driver.py +++ b/packetary/tests/test_deb_driver.py @@ -26,16 +26,12 @@ from packetary.tests import base from packetary.tests.stubs.generator import gen_package from packetary.tests.stubs.generator import gen_repository from packetary.tests.stubs.helpers import get_compressed +from packetary.tests.stubs.helpers import HTTPError PACKAGES = path.join(path.dirname(__file__), "data", "Packages") -class HTTPError(Exception): - def __init__(self, code): - self.code = code - - class TestDebDriver(base.TestCase): @classmethod def setUpClass(cls): @@ -318,7 +314,9 @@ class TestDebDriver(base.TestCase): "section": "main", "type": "rpm", "priority": "100", "origin": "Origin", "path": "/repo" } - repo = self.driver.create_repository(repository_data, "x86_64") + repo = self.driver.create_repository( + self.connection, repository_data, "x86_64" + ) self.assertEqual(repository_data["name"], repo.name) self.assertEqual("x86_64", repo.architecture) self.assertEqual(repository_data["uri"] + "/", repo.url) @@ -346,10 +344,14 @@ class TestDebDriver(base.TestCase): "origin": "Origin", "path": "/repo" } with self.assertRaisesRegexp(ValueError, "flat format"): - self.driver.create_repository(repository_data, "x86_64") + self.driver.create_repository( + self.connection, repository_data, "x86_64" + ) with self.assertRaisesRegexp(ValueError, "single component"): repository_data["section"] = ["main", "universe"] - self.driver.create_repository(repository_data, "x86_64") + self.driver.create_repository( + self.connection, repository_data, "x86_64" + ) @mock.patch.multiple( "packetary.drivers.deb_driver", diff --git a/packetary/tests/test_repository_contoller.py b/packetary/tests/test_repository_contoller.py index d6816c8..e07169c 100644 --- a/packetary/tests/test_repository_contoller.py +++ b/packetary/tests/test_repository_contoller.py @@ -161,7 +161,7 @@ class TestRepositoryController(base.TestCase): self.driver.get_relative_path.side_effect = ["pool/t/test1.pkg"] self.ctrl.create_repository(repository_data, packages_list) self.driver.create_repository.assert_called_once_with( - repository_data, self.ctrl.arch + self.context.connection, repository_data, self.ctrl.arch ) self.driver.get_relative_path.assert_called_once_with( repo, "test1.pkg" diff --git a/packetary/tests/test_rpm_driver.py b/packetary/tests/test_rpm_driver.py index 72ffdc2..e1d42d3 100644 --- a/packetary/tests/test_rpm_driver.py +++ b/packetary/tests/test_rpm_driver.py @@ -171,12 +171,12 @@ class TestRpmDriver(base.TestCase): self.assertTrue(package.mandatory) @mock.patch("packetary.drivers.rpm_driver.os") - @mock.patch("packetary.drivers.rpm_driver.tempfile.NamedTemporaryFile") - def test_add_packages_to_existing(self, tmp_mock, os_mock): + @mock.patch("packetary.drivers.rpm_driver.open") + def test_add_packages_to_existing(self, open_mock, os_mock): self.configure_streams() tmp_file = mock.MagicMock() tmp_file.name = "/tmp/groups.gz" - tmp_mock.return_value.__enter__.return_value = tmp_file + open_mock.return_value.__enter__.return_value = tmp_file repo = gen_repository("test", url="file:///repo/os/x86_64") md_gen = mock.MagicMock() self.createrepo.MetaDataGenerator.return_value = md_gen @@ -184,6 +184,11 @@ class TestRpmDriver(base.TestCase): md_gen.tempdir = "tmp" md_gen.finaldir = "repodata" os_mock.path.exists.return_value = True + os_mock.path.join.side_effect = [ + "/repo/os/x86_64/tmp", + tmp_file.name, + "/repo/os/x86_64/tmp" + ] self.driver.add_packages(self.connection, repo, set()) self.assertEqual( "/repo/os/x86_64", @@ -202,14 +207,14 @@ class TestRpmDriver(base.TestCase): @mock.patch("packetary.drivers.rpm_driver.os") @mock.patch("packetary.drivers.rpm_driver.shutil") - @mock.patch("packetary.drivers.rpm_driver.tempfile.NamedTemporaryFile") + @mock.patch("packetary.drivers.rpm_driver.open") def test_add_packages_clean_metadata_on_error( - self, tmp_mock, shutil_mock, os_mock + self, open_mock, shutil_mock, os_mock ): self.configure_streams() tmp_file = mock.MagicMock() tmp_file.name = "/tmp/groups.gz" - tmp_mock.return_value.__enter__.return_value = tmp_file + open_mock.return_value.__enter__.return_value = tmp_file self.createrepo.MDError = ValueError md_gen = mock.MagicMock() self.createrepo.MetaDataGenerator.return_value = md_gen @@ -220,7 +225,11 @@ class TestRpmDriver(base.TestCase): self.createrepo.MetaDataConfig().tempdir = "tmp" self.createrepo.MetaDataConfig().finaldir = "repodata" os_mock.path.exists.return_value = True - os_mock.path.join.side_effect = lambda *a: '/'.join(a) + os_mock.path.join.side_effect = [ + "/repo/os/x86_64/tmp", + tmp_file.name, + "/repo/os/x86_64/tmp" + ] with self.assertRaises(RuntimeError): self.driver.add_packages(self.connection, repo, set()) shutil_mock.rmtree.assert_called_once_with( @@ -229,15 +238,15 @@ class TestRpmDriver(base.TestCase): os_mock.unlink.assert_called_once_with(tmp_file.name) @mock.patch("packetary.drivers.rpm_driver.os") - @mock.patch("packetary.drivers.rpm_driver.tempfile.NamedTemporaryFile") + @mock.patch("packetary.drivers.rpm_driver.open") @mock.patch("packetary.drivers.rpm_driver.utils.ensure_dir_exist") - def test_fork_repository( - self, ensure_dir_exists_mock, tmp_mock, os_mock + def test_fork_repository_to_empty_destination( + self, m_ensure_dir_exists, m_open, m_os ): self.configure_streams() tmp_file = mock.MagicMock() tmp_file.name = "/tmp/groups.gz" - tmp_mock.return_value.__enter__.return_value = tmp_file + m_open.return_value.__enter__.return_value = tmp_file repo = gen_repository("os", url="http://localhost/os/x86_64/") md_gen = mock.MagicMock() self.createrepo.MetaDataGenerator.return_value = md_gen @@ -247,12 +256,18 @@ class TestRpmDriver(base.TestCase): md_gen.finaldir = "repodata" md_config = mock.MagicMock() self.createrepo.MetaDataConfig.return_value = md_config + m_os.path.join.side_effect = [ + "/repo/os/x86_64/repodata/repomd.xml", + tmp_file.name, + "/repo/os/x86_64/tmp" + ] + m_os.path.exists.return_value = False new_repo = self.driver.fork_repository( self.connection, repo, "/repo/os/x86_64" ) - ensure_dir_exists_mock.assert_called_once_with("/repo/os/x86_64") + m_ensure_dir_exists.assert_called_once_with("/repo/os/x86_64") self.assertEqual(repo.name, new_repo.name) self.assertEqual(repo.architecture, new_repo.architecture) self.assertEqual("file:///repo/os/x86_64/", new_repo.url) @@ -261,13 +276,51 @@ class TestRpmDriver(base.TestCase): self.assertEqual(["*"], md_config.excludes) self.assertFalse(md_config.update) self.assertEqual(tmp_file.name, md_config.groupfile) - os_mock.unlink.assert_called_once_with(tmp_file.name) + m_open.assert_called_once_with(tmp_file.name, "w") + m_os.unlink.assert_called_once_with(tmp_file.name) @mock.patch("packetary.drivers.rpm_driver.os") - @mock.patch("packetary.drivers.rpm_driver.tempfile.NamedTemporaryFile") + @mock.patch("packetary.drivers.rpm_driver.open") + @mock.patch("packetary.drivers.rpm_driver.utils.ensure_dir_exist") + def test_fork_repository_to_existing_destination( + self, ensure_dir_exists_mock, open_mock, os_mock + ): + self.configure_streams() + tmp_file = mock.MagicMock() + tmp_file.name = "/tmp/groups.gz" + open_mock.return_value.__enter__.return_value = tmp_file + repo = gen_repository("os", url="http://localhost/os/x86_64/") + md_gen = mock.MagicMock() + self.createrepo.MetaDataGenerator.return_value = md_gen + md_gen.doFinalMove.side_effect = [None] + md_gen.outputdir = "/repo/os/x86_64" + md_gen.tempdir = "tmp" + md_gen.finaldir = "repodata" + md_config = mock.MagicMock() + self.createrepo.MetaDataConfig.return_value = md_config + os_mock.path.join.side_effect = [ + "/repo/os/x86_64/repodata/repomd.xml", + tmp_file.name, + "/repo/os/x86_64/tmp" + ] + os_mock.path.exists.return_value = True + groups_xml = mock.MagicMock() + with mock.patch.object(self.driver, "_load_groups") as load_mock: + load_mock.side_effect = [None, groups_xml] + self.driver.fork_repository( + self.connection, + repo, + "/repo/os/x86_64" + ) + self.assertEqual(tmp_file.name, md_config.groupfile) + os_mock.unlink.assert_called_once_with(tmp_file.name) + groups_xml.write.assert_called_once_with(tmp_file) + + @mock.patch("packetary.drivers.rpm_driver.os") + @mock.patch("packetary.drivers.rpm_driver.open") @mock.patch("packetary.drivers.rpm_driver.utils.ensure_dir_exist") def test_create_repository( - self, ensure_dir_exists_mock, tmp_mock, os_mock + self, ensure_dir_exists_mock, open_mock, os_mock ): repository_data = { "name": "Test", "uri": "file:///repo/os/x86_64", "origin": "Test", @@ -283,7 +336,9 @@ class TestRpmDriver(base.TestCase): md_config = mock.MagicMock() self.createrepo.MetaDataConfig.return_value = md_config - repo = self.driver.create_repository(repository_data, "x86_64") + repo = self.driver.create_repository( + self.connection, repository_data, "x86_64" + ) ensure_dir_exists_mock.assert_called_once_with("/repo/os/x86_64/") self.assertEqual(repository_data["name"], repo.name) self.assertEqual("x86_64", repo.architecture) @@ -293,6 +348,8 @@ class TestRpmDriver(base.TestCase): md_gen.doFinalMove.assert_called_once_with() self.assertEqual(["*"], md_config.excludes) self.assertFalse(md_config.update) + open_mock.assert_not_called() + os_mock.unlink.assert_not_called() @mock.patch("packetary.drivers.rpm_driver.utils") def test_load_package_from_file(self, utils): diff --git a/setup.py b/setup.py index ec4d54a..a83280e 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,7 @@ # THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT import setuptools +import pkg_resources # In python < 2.7.4, a lazy loading of package `pbr` will break # setuptools if some other modules registered functions in `atexit`.