diff --git a/packetary/__init__.py b/packetary/__init__.py index bb8ecae..2fca754 100644 --- a/packetary/__init__.py +++ b/packetary/__init__.py @@ -33,7 +33,4 @@ try: __version__ = pbr.version.VersionInfo( 'packetary').version_string() except Exception as e: - # when run tests without installing package - # pbr may raise exception. - print("ERROR:", e) __version__ = "0.0.0" diff --git a/packetary/api.py b/packetary/api.py index 5b9f7c5..2da17a7 100644 --- a/packetary/api.py +++ b/packetary/api.py @@ -18,6 +18,7 @@ from collections import defaultdict import logging +import re import jsonschema import six @@ -30,6 +31,7 @@ from packetary.objects import PackagesForest from packetary.objects import PackagesTree from packetary.objects.statistics import CopyStatistics from packetary.schemas import PACKAGE_FILES_SCHEMA +from packetary.schemas import PACKAGE_FILTER_SCHEMA from packetary.schemas import PACKAGES_SCHEMA logger = logging.getLogger(__package__) @@ -128,22 +130,27 @@ class RepositoryApi(object): return self.controller.create_repository(repo_data, package_files) def get_packages(self, repos_data, requirements_data=None, - include_mandatory=False): + include_mandatory=False, filter_data=None): """Gets the list of packages from repository(es). :param repos_data: The list of repository descriptions :param requirements_data: The list of package`s requirements that should be included :param include_mandatory: if True, all mandatory packages will be + included + :param filter_data: A set of filters that is used to exclude + those packages which match one of filters :return: the set of packages """ repos = self._load_repositories(repos_data) requirements = self._load_requirements(requirements_data) - return self._get_packages(repos, requirements, include_mandatory) + exclude_filter = self._load_filter(filter_data) + return self._get_packages(repos, requirements, + include_mandatory, exclude_filter) def clone_repositories(self, repos_data, requirements_data, destination, include_source=False, include_locale=False, - include_mandatory=False): + include_mandatory=False, filter_data=None): """Creates the clones of specified repositories in local folder. :param repos_data: The list of repository descriptions @@ -155,12 +162,16 @@ class RepositoryApi(object): :param include_locale: if True, the locales will be copied as well. :param include_mandatory: if True, all mandatory packages will be included + :param filter_data: A set of filters that is used to exclude + those packages which match one of filters :return: count of copied and total packages. """ repos = self._load_repositories(repos_data) reqs = self._load_requirements(requirements_data) - all_packages = self._get_packages(repos, reqs, include_mandatory) + exclude_filter = self._load_filter(filter_data) + all_packages = self._get_packages( + repos, reqs, include_mandatory, exclude_filter) package_groups = defaultdict(set) for pkg in all_packages: package_groups[pkg.repository].add(pkg) @@ -191,7 +202,8 @@ class RepositoryApi(object): self._load_packages(self._load_repositories(repos_data), packages.add) return packages.get_unresolved_dependencies() - def _get_packages(self, repos, requirements, include_mandatory): + def _get_packages(self, repos, requirements, include_mandatory, + exclude_filter): if requirements is not None: forest = PackagesForest() for repo in repos: @@ -199,7 +211,12 @@ class RepositoryApi(object): return forest.get_packages(requirements, include_mandatory) packages = set() - self._load_packages(repos, packages.add) + consumer = packages.add + if exclude_filter is not None: + def consumer(p): + if not exclude_filter(p): + packages.add(p) + self._load_packages(repos, consumer) return packages def _load_packages(self, repos, consumer): @@ -228,6 +245,51 @@ class RepositoryApi(object): )) return result + def _load_filter(self, filter_data): + """Loads filter from filter data. + + Property value could be a string or a python regexp. + Example of filters data: + - name: full-package-name + section: section1 + - name: /^.*substr/ + + :param filter_data: A list of filters + :return: Lambda that could match a particular package. + """ + + if filter_data is None: + return + + self._validate_filter_data(filter_data) + + def get_pattern_match(pattern, key, value): + return lambda p: pattern.match(getattr(p, key)) + + def get_exact_match(key, value): + return lambda p: getattr(p, key) == value + + def get_logical_and(filters): + return lambda p: all((f(p) for f in filters)) + + def get_logical_or(filters): + return lambda p: any((f(p) for f in filters)) + + filters = [] + for fdata in filter_data: + matchers = [] + for key, value in six.iteritems(fdata): + if value.startswith('/') and value.endswith('/'): + pattern = re.compile(value[1:-1]) + matchers.append(get_pattern_match(pattern, key, value)) + else: + matchers.append(get_exact_match(key, value)) + filters.append(get_logical_and(matchers)) + return get_logical_or(filters) + + def _validate_filter_data(self, filter_data): + self._validate_data(filter_data, PACKAGE_FILTER_SCHEMA) + def _validate_repo_data(self, repo_data): schema = self.controller.get_repository_data_schema() self._validate_data(repo_data, schema) diff --git a/packetary/cli/commands/base.py b/packetary/cli/commands/base.py index 301c3b8..8ebe639 100644 --- a/packetary/cli/commands/base.py +++ b/packetary/cli/commands/base.py @@ -111,7 +111,9 @@ class PackagesMixin(object): help="Do not copy mandatory packages." ) - parser.add_argument( + group = parser.add_mutually_exclusive_group() + + group.add_argument( "-p", "--packages", dest='requirements', type=read_from_file, @@ -119,6 +121,15 @@ class PackagesMixin(object): help="The path to file with list of packages." "See documentation about format." ) + + group.add_argument( + "-f", "--exclude-filter", + dest='exclude_filter_data', + type=read_from_file, + metavar='FILENAME', + help="The path to file with package exclude filter data." + "See documentation about format." + ) return parser diff --git a/packetary/cli/commands/clone.py b/packetary/cli/commands/clone.py index 5131330..4afed94 100644 --- a/packetary/cli/commands/clone.py +++ b/packetary/cli/commands/clone.py @@ -54,7 +54,8 @@ class CloneCommand(PackagesMixin, RepositoriesMixin, BaseRepoCommand): parsed_args.destination, parsed_args.sources, parsed_args.locales, - parsed_args.include_mandatory + parsed_args.include_mandatory, + filter_data=parsed_args.exclude_filter_data, ) self.stdout.write( "Packages copied: {0.copied}/{0.total}.\n".format(stat) diff --git a/packetary/cli/commands/packages.py b/packetary/cli/commands/packages.py index b3fc82d..9ee8987 100644 --- a/packetary/cli/commands/packages.py +++ b/packetary/cli/commands/packages.py @@ -41,7 +41,8 @@ class ListOfPackages( return api.get_packages( parsed_args.repositories, parsed_args.requirements, - parsed_args.include_mandatory + parsed_args.include_mandatory, + filter_data=parsed_args.exclude_filter_data, ) diff --git a/packetary/drivers/deb_driver.py b/packetary/drivers/deb_driver.py index bc5b362..2a9601c 100644 --- a/packetary/drivers/deb_driver.py +++ b/packetary/drivers/deb_driver.py @@ -157,6 +157,7 @@ class DebRepositoryDriver(RepositoryDriverBase): # The deb does not have obsoletes section obsoletes=[], provides=self._get_relations(dpkg, "provides"), + group=dpkg.get("section"), )) except KeyError as e: self.logger.error( @@ -255,7 +256,8 @@ class DebRepositoryDriver(RepositoryDriverBase): "recommends" ), provides=self._get_relations(debcontrol, "provides"), - obsoletes=[] + obsoletes=[], + group=debcontrol.get('section'), ) def get_relative_path(self, repository, filename): diff --git a/packetary/drivers/rpm_driver.py b/packetary/drivers/rpm_driver.py index 5752d75..a69e64a 100644 --- a/packetary/drivers/rpm_driver.py +++ b/packetary/drivers/rpm_driver.py @@ -144,7 +144,9 @@ class RpmRepositoryDriver(RepositoryDriverBase): mandatory=name in mandatory, requires=self._get_relations(tag, "requires"), obsoletes=self._get_relations(tag, "obsoletes"), - provides=self._get_relations(tag, "provides") + provides=self._get_relations(tag, "provides"), + group=tag.find("./main:format/rpm:group", + _NAMESPACES).text, )) except (ValueError, KeyError) as e: self.logger.error( @@ -226,6 +228,7 @@ class RpmRepositoryDriver(RepositoryDriverBase): requires=self._parse_package_relations(pkg.requires), obsoletes=self._parse_package_relations(pkg.obsoletes), provides=self._parse_package_relations(pkg.provides), + group=hdr["group"], ) def get_relative_path(self, repository, filename): diff --git a/packetary/objects/package.py b/packetary/objects/package.py index 145b5e5..3a36872 100644 --- a/packetary/objects/package.py +++ b/packetary/objects/package.py @@ -29,7 +29,8 @@ class Package(ComparableObject): def __init__(self, repository, name, version, filename, filesize, checksum, mandatory=False, - requires=None, provides=None, obsoletes=None): + requires=None, provides=None, obsoletes=None, + group=None): """Initialises. :param name: the package`s name @@ -41,6 +42,7 @@ class Package(ComparableObject): :param provides: the package`s provides(optional) :param obsoletes: the package`s obsoletes(optional) :param mandatory: indicates that package is mandatory + :param group: corresponds to rpm group and deb section """ self.repository = repository @@ -53,6 +55,7 @@ class Package(ComparableObject): self.provides = provides or [] self.obsoletes = obsoletes or [] self.mandatory = mandatory + self.group = group def __copy__(self): """Creates shallow copy of package.""" diff --git a/packetary/schemas/__init__.py b/packetary/schemas/__init__.py index ab1922c..f89036f 100644 --- a/packetary/schemas/__init__.py +++ b/packetary/schemas/__init__.py @@ -18,12 +18,14 @@ from packetary.schemas.deb_repo_schema import DEB_REPO_SCHEMA from packetary.schemas.package_files_schema import PACKAGE_FILES_SCHEMA +from packetary.schemas.package_filter_schema import PACKAGE_FILTER_SCHEMA from packetary.schemas.packages_schema import PACKAGES_SCHEMA from packetary.schemas.rpm_repo_schema import RPM_REPO_SCHEMA __all__ = [ "DEB_REPO_SCHEMA", + "PACKAGE_FILES_SCHEMA", + "PACKAGE_FILTER_SCHEMA", "PACKAGES_SCHEMA", "RPM_REPO_SCHEMA", - "PACKAGE_FILES_SCHEMA" ] diff --git a/packetary/schemas/package_filter_schema.py b/packetary/schemas/package_filter_schema.py new file mode 100644 index 0000000..e1b01e8 --- /dev/null +++ b/packetary/schemas/package_filter_schema.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- + +# Copyright 2016 Mirantis, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +PACKAGE_FILTER_SCHEMA = { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "group": { + "type": "string" + } + } + } +} diff --git a/packetary/tests/test_cli_commands.py b/packetary/tests/test_cli_commands.py index a210191..0b6e139 100644 --- a/packetary/tests/test_cli_commands.py +++ b/packetary/tests/test_cli_commands.py @@ -108,7 +108,10 @@ class TestCliCommands(base.TestCase): read_file_mock.assert_any_call("packages.yaml") api_instance.clone_repositories.assert_called_once_with( [{"name": "repo"}], [{"name": "package"}], "/root", - False, False, False + False, + False, + False, + filter_data=None, ) stdout_mock.write.assert_called_once_with( "Packages copied: 0/0.\n" @@ -129,7 +132,7 @@ class TestCliCommands(base.TestCase): ) self.check_common_config(api_mock.create.call_args[0][0]) api_instance.get_packages.assert_called_once_with( - [{"name": "repo"}], None, True + [{"name": "repo"}], None, True, filter_data=None ) self.assertIn( "test1; test1.pkg", diff --git a/packetary/tests/test_repository_api.py b/packetary/tests/test_repository_api.py index 64a18d4..23ecd0e 100644 --- a/packetary/tests/test_repository_api.py +++ b/packetary/tests/test_repository_api.py @@ -25,6 +25,7 @@ from packetary.api import Configuration from packetary.api import Context from packetary.api import RepositoryApi from packetary.schemas import PACKAGE_FILES_SCHEMA +from packetary.schemas import PACKAGE_FILTER_SCHEMA from packetary.schemas import PACKAGES_SCHEMA from packetary.tests import base from packetary.tests.stubs import generator @@ -119,7 +120,7 @@ class TestRepositoryApi(base.TestCase): ) def test_get_packages_as_is(self, jsonschema_mock): - packages = self.api.get_packages([self.repo_data], None) + packages = self.api.get_packages([self.repo_data], None, False, None) self.assertEqual(5, len(packages)) self.assertItemsEqual( self.packages, @@ -133,7 +134,7 @@ class TestRepositoryApi(base.TestCase): jsonschema_mock): requirements = [{"name": "package1"}] packages = self.api.get_packages( - [self.repo_data], requirements, True + [self.repo_data], requirements, True, None ) self.assertEqual(3, len(packages)) self.assertItemsEqual( @@ -151,7 +152,7 @@ class TestRepositoryApi(base.TestCase): jsonschema_mock): requirements = [{"name": "package4"}] packages = self.api.get_packages( - [self.repo_data], requirements, False + [self.repo_data], requirements, False, None ) self.assertEqual(2, len(packages)) self.assertItemsEqual( @@ -239,6 +240,42 @@ class TestRepositoryApi(base.TestCase): ] ) + def test_clone_with_filter(self, jsonschema_mock): + repos_data = "repos_data" + requirements_data = "requirements_data" + filter_data = "filter_data" + repos = "repos" + requirements = "requirements" + exclude_filter = "exclude_filter" + + self.api._load_repositories = mock.Mock(return_value=repos) + self.api._load_requirements = mock.Mock(return_value=requirements) + self.api._load_filter = mock.Mock(return_value=exclude_filter) + self.api._get_packages = mock.Mock(return_value=set()) + self.api.controller = mock.Mock() + + self.api.clone_repositories(repos_data, requirements_data, + "destination", filter_data=filter_data) + + self.api._load_repositories.assert_called_once_with(repos_data) + self.api._load_requirements.assert_called_once_with(requirements_data) + self.api._load_filter.assert_called_once_with(filter_data) + self.api._get_packages.assert_called_once_with( + repos, requirements, False, exclude_filter) + + def test_get_packages_with_exclude_filter(self, jsonschema_mock): + exclude_filter = lambda p: any([p == "p1", p == "p3"]) + self.api._load_packages = CallbacksAdapter() + self.api._load_packages.return_value = ["p1", "p2", "p3", "p4"] + packages = self.api._get_packages("repos", None, False, exclude_filter) + self.assertSetEqual(packages, set(["p2", "p4"])) + + def test_get_packages_without_exclude_filter(self, jsonschema_mock): + self.api._load_packages = CallbacksAdapter() + self.api._load_packages.return_value = ["p1", "p2"] + packages = self.api._get_packages("repos", None, False, None) + self.assertSetEqual(packages, set(["p1", "p2"])) + def test_get_unresolved(self, jsonschema_mock): unresolved = self.api.get_unresolved_dependencies([self.repo_data]) self.assertItemsEqual(["package6"], (x.name for x in unresolved)) @@ -246,6 +283,43 @@ class TestRepositoryApi(base.TestCase): self.repo_data, self.schema ) + def test_load_filter_with_none(self, jsonschema_mock): + self.assertIsNone(self.api._load_filter(None)) + + def test_load_filter(self, jsonschema_mock): + self.api._validate_filter_data = mock.Mock() + filter_data = [ + {"name": "p1", "group": "g1"}, + {"name": "p2"}, + {"group": "g3"}, + {"name": "/^.5/", "group": "/^.*3/"}, + {"group": "/^.*4/"}, + ] + exclude_filter = self.api._load_filter(filter_data) + + p1 = generator.gen_package(name="p1", group="g1") + p2 = generator.gen_package(name="p2", group="g1") + p3 = generator.gen_package(name="p3", group="g2") + p4 = generator.gen_package(name="p4", group="g3") + p5 = generator.gen_package(name="p5", group="g3") + p6 = generator.gen_package(name="p6", group="g4") + + cases = [ + (True, (p1,)), + (True, (p2,)), + (False, (p3,)), + (True, (p4,)), + (True, (p5,)), + (True, (p6,)), + ] + self._check_cases(self.assertEqual, cases, exclude_filter) + + def test_validate_filter_data(self, jsonschema_mock): + self.api._validate_data = mock.Mock() + self.api._validate_filter_data("filter_data") + self.api._validate_data.assert_called_once_with("filter_data", + PACKAGE_FILTER_SCHEMA) + def test_load_requirements(self, jsonschema_mock): expected = { generator.gen_relation("test1"), diff --git a/packetary/tests/test_rpm_driver.py b/packetary/tests/test_rpm_driver.py index 07d9296..6155c38 100644 --- a/packetary/tests/test_rpm_driver.py +++ b/packetary/tests/test_rpm_driver.py @@ -41,9 +41,9 @@ GROUPS_DB = path.join(path.dirname(__file__), "data", "groups.xml") class TestRpmDriver(base.TestCase): @classmethod def setUpClass(cls): - cls.createrepo = sys.modules["createrepo"] = mock.MagicMock() - # import driver class after patching sys.modules + sys.modules["createrepo"] = mock.MagicMock() from packetary.drivers import rpm_driver + cls.createrepo = rpm_driver.createrepo = mock.MagicMock() super(TestRpmDriver, cls).setUpClass() cls.driver = rpm_driver.RpmRepositoryDriver() @@ -243,7 +243,7 @@ class TestRpmDriver(base.TestCase): self.createrepo.yumbased.YumLocalPackage.return_value = rpm_mock rpm_mock.returnLocalHeader.return_value = { "name": "Test", "epoch": 1, "version": "1.2.3", "release": "1", - "size": "10" + "size": "10", "group": "Group" } repo = gen_repository("Test", url="file:///repo/os/x86_64/") pkg = self.driver.load_package_from_file(repo, "test.rpm") @@ -261,6 +261,7 @@ class TestRpmDriver(base.TestCase): self.assertEqual("1-1.2.3-1", str(pkg.version)) self.assertEqual("test.rpm", pkg.filename) self.assertEqual((3, 4, 5), pkg.checksum) + self.assertEqual("Group", pkg.group) self.assertEqual(10, pkg.filesize) self.assertItemsEqual( ['test1 (= 0-1.2.3-1.el5)'], diff --git a/requirements.txt b/requirements.txt index a281751..f50a483 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ eventlet>=0.15 bintrees>=2.0.2 chardet>=2.0.1 stevedore>=1.1.0 -six>=1.5.2 +six>=1.9.0 # MIT python-debian>=0.1.21 lxml>=3.2 jsonschema!=2.5.0,<3.0.0,>=2.0.0 # MIT diff --git a/tox.ini b/tox.ini index 3e02f93..0594aa7 100644 --- a/tox.ini +++ b/tox.ini @@ -5,6 +5,7 @@ skipsdist = True [testenv] usedevelop = True +sitepackages = True install_command = pip install -U {opts} {packages} setenv = VIRTUAL_ENV={envdir}