x
/
packetary
Code Issues Proposed changes

Introduced new scheme to declare requirements 

the requirements can contain the following sections:
packages - the list of packages that are needed
repositories - the list of repositories, packages from that are needed
mandatory - boolean flag that uses to automatically copy mandatory packages

Change-Id: Ic26f991c0bf1e9819005cd4bbe7ed40228b2ce1b
This commit is contained in:
Bulat Gaifullin 2016-04-24 08:57:41 -05:00
parent 4569ca760f
commit d661055322
32 changed files with 1050 additions and 825 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
packetary/__init__.py
View File

@ -18,19 +18,13 @@
import pbr.version
from packetary.api import Configuration
from packetary.api import Context
from packetary.api import RepositoryApi
from packetary import api
__all__ = [
"Configuration",
"Context",
"RepositoryApi",
]
__all__ = ["api", "__version__"]
try:
__version__ = pbr.version.VersionInfo(
'packetary').version_string()
except Exception as e:
except Exception:
__version__ = "0.0.0"

328
packetary/api.py
View File

@ -1,328 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mirantis, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
from collections import defaultdict
import logging
import re
import jsonschema
import six
from packetary.controllers import RepositoryController
from packetary.library.connections import ConnectionsManager
from packetary.library.executor import AsynchronousSection
from packetary.objects import PackageRelation
from packetary.objects import PackagesForest
from packetary.objects import PackagesTree
from packetary.objects.statistics import CopyStatistics
from packetary.schemas import PACKAGE_FILES_SCHEMA
from packetary.schemas import PACKAGE_FILTER_SCHEMA
from packetary.schemas import PACKAGES_SCHEMA
logger = logging.getLogger(__package__)
class Configuration(object):
"""The configuration holder."""
def __init__(self, http_proxy=None, https_proxy=None,
retries_num=0, retry_interval=0, threads_num=0,
ignore_errors_num=0):
"""Initialises.
:param http_proxy: the url of proxy for connections over http,
no-proxy will be used if it is not specified
:param https_proxy: the url of proxy for connections over https,
no-proxy will be used if it is not specified
:param retries_num: the number of retries on errors
:param retry_interval: the minimal time between retries (in seconds)
:param threads_num: the max number of active threads
:param ignore_errors_num: the number of errors that may occurs
before stop processing
"""
self.http_proxy = http_proxy
self.https_proxy = https_proxy
self.ignore_errors_num = ignore_errors_num
self.retries_num = retries_num
self.retry_interval = retry_interval
self.threads_num = threads_num
class Context(object):
"""The infra-objects holder."""
def __init__(self, config):
"""Initialises.
:param config: the configuration
"""
self._connection = ConnectionsManager(
proxy=config.http_proxy,
secure_proxy=config.https_proxy,
retries_num=config.retries_num,
retry_interval=config.retry_interval
)
self._threads_num = config.threads_num
self._ignore_errors_num = config.ignore_errors_num
@property
def connection(self):
"""Gets the connection."""
return self._connection
def async_section(self, ignore_errors_num=None):
"""Gets the execution scope.
:param ignore_errors_num: custom value for ignore_errors_num,
the class value is used if omitted.
"""
if ignore_errors_num is None:
ignore_errors_num = self._ignore_errors_num
return AsynchronousSection(self._threads_num, ignore_errors_num)
class RepositoryApi(object):
"""Provides high-level API to operate with repositories."""
def __init__(self, controller):
"""Initialises.
:param controller: the repository controller.
"""
self.controller = controller
@classmethod
def create(cls, config, repotype, repoarch):
"""Creates the repository API instance.
:param config: the configuration
:param repotype: the kind of repository(deb, yum, etc)
:param repoarch: the architecture of repository (x86_64 or i386)
"""
context = config if isinstance(config, Context) else Context(config)
return cls(RepositoryController.load(context, repotype, repoarch))
def create_repository(self, repo_data, package_files):
"""Create new repository with specified packages.
:param repo_data: The description of repository
:param package_files: The list of URLs of packages
"""
self._validate_repo_data(repo_data)
self._validate_package_files(package_files)
return self.controller.create_repository(repo_data, package_files)
def get_packages(self, repos_data, requirements_data=None,
include_mandatory=False, filter_data=None):
"""Gets the list of packages from repository(es).
:param repos_data: The list of repository descriptions
:param requirements_data: The list of package`s requirements
that should be included
:param include_mandatory: if True, all mandatory packages will be
included
:param filter_data: A set of filters that is used to exclude
those packages which match one of filters
:return: the set of packages
"""
repos = self._load_repositories(repos_data)
requirements = self._load_requirements(requirements_data)
exclude_filter = self._load_filter(filter_data)
return self._get_packages(repos, requirements,
include_mandatory, exclude_filter)
def clone_repositories(self, repos_data, requirements_data, destination,
include_source=False, include_locale=False,
include_mandatory=False, filter_data=None):
"""Creates the clones of specified repositories in local folder.
:param repos_data: The list of repository descriptions
:param requirements_data: The list of package`s requirements
that should be included
:param destination: the destination folder path
:param include_source: if True, the source packages
will be copied as well.
:param include_locale: if True, the locales will be copied as well.
:param include_mandatory: if True, all mandatory packages will be
included
:param filter_data: A set of filters that is used to exclude
those packages which match one of filters
:return: count of copied and total packages.
"""
repos = self._load_repositories(repos_data)
reqs = self._load_requirements(requirements_data)
exclude_filter = self._load_filter(filter_data)
all_packages = self._get_packages(
repos, reqs, include_mandatory, exclude_filter)
package_groups = defaultdict(set)
for pkg in all_packages:
package_groups[pkg.repository].add(pkg)
stat = CopyStatistics()
mirrors = defaultdict(set)
# group packages by mirror
for repo, packages in six.iteritems(package_groups):
mirror = self.controller.fork_repository(
repo, destination, include_source, include_locale
)
mirrors[mirror].update(packages)
# add new packages to mirrors
for mirror, packages in six.iteritems(mirrors):
self.controller.assign_packages(
mirror, packages, stat.on_package_copied
)
return stat
def get_unresolved_dependencies(self, repos_data):
"""Gets list of unresolved dependencies for repository(es).
:param repos_data: The list of repository descriptions
:return: list of unresolved dependencies
"""
packages = PackagesTree()
self._load_packages(self._load_repositories(repos_data), packages.add)
return packages.get_unresolved_dependencies()
def _get_packages(self, repos, requirements, include_mandatory,
exclude_filter):
if requirements is not None:
forest = PackagesForest()
for repo in repos:
self.controller.load_packages(repo, forest.add_tree().add)
return forest.get_packages(requirements, include_mandatory)
packages = set()
consumer = packages.add
if exclude_filter is not None:
def consumer(p):
if not exclude_filter(p):
packages.add(p)
self._load_packages(repos, consumer)
return packages
def _load_packages(self, repos, consumer):
for repo in repos:
self.controller.load_packages(repo, consumer)
def _load_repositories(self, repos_data):
for repo_data in repos_data:
self._validate_repo_data(repo_data)
return self.controller.load_repositories(repos_data)
def _load_requirements(self, requirements_data):
if requirements_data is None:
return
self._validate_requirements_data(requirements_data)
result = []
for r in requirements_data:
versions = r.get('versions', None)
if versions is None:
result.append(PackageRelation.from_args((r['name'],)))
else:
for version in versions:
result.append(PackageRelation.from_args(
([r['name']] + version.split(None, 1))
))
return result
def _load_filter(self, filter_data):
"""Loads filter from filter data.
Property value could be a string or a python regexp.
Example of filters data:
- name: full-package-name
section: section1
- name: /^.*substr/
:param filter_data: A list of filters
:return: Lambda that could match a particular package.
"""
if filter_data is None:
return
self._validate_filter_data(filter_data)
def get_pattern_match(pattern, key, value):
return lambda p: pattern.match(getattr(p, key))
def get_exact_match(key, value):
return lambda p: getattr(p, key) == value
def get_logical_and(filters):
return lambda p: all((f(p) for f in filters))
def get_logical_or(filters):
return lambda p: any((f(p) for f in filters))
filters = []
for fdata in filter_data:
matchers = []
for key, value in six.iteritems(fdata):
if value.startswith('/') and value.endswith('/'):
pattern = re.compile(value[1:-1])
matchers.append(get_pattern_match(pattern, key, value))
else:
matchers.append(get_exact_match(key, value))
filters.append(get_logical_and(matchers))
return get_logical_or(filters)
def _validate_filter_data(self, filter_data):
self._validate_data(filter_data, PACKAGE_FILTER_SCHEMA)
def _validate_repo_data(self, repo_data):
schema = self.controller.get_repository_data_schema()
self._validate_data(repo_data, schema)
def _validate_requirements_data(self, requirements_data):
self._validate_data(requirements_data, PACKAGES_SCHEMA)
def _validate_package_files(self, package_files):
self._validate_data(package_files, PACKAGE_FILES_SCHEMA)
def _validate_data(self, data, schema):
"""Validate the input data using jsonschema validation.
:param data: a data to validate represented as a dict
:param schema: a schema to validate represented as a dict;
must be in JSON Schema Draft 4 format.
"""
try:
jsonschema.validate(data, schema)
except jsonschema.ValidationError as e:
self._raise_validation_error(
"data", e.message, e.path
)
except jsonschema.SchemaError as e:
self._raise_validation_error(
"schema", e.message, e.schema_path
)
@staticmethod
def _raise_validation_error(what, details, path):
message = "Invalid {0}: {1}.".format(what, details)
if path:
message += "\nField: [{0}]".format(
"][".join(repr(p) for p in path)
)
raise ValueError(message)

26
packetary/schemas/package_filter_schema.py → packetary/api/__init__.py
View File

@ -16,18 +16,14 @@
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
PACKAGE_FILTER_SCHEMA = {
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "array",
"items": {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"group": {
"type": "string"
}
}
}
}
from packetary.api.context import Configuration
from packetary.api.context import Context
from packetary.api.options import RepositoryCopyOptions
from packetary.api.repositories import RepositoryApi
__all__ = [
"Configuration",
"Context",
"RepositoryApi",
"RepositoryCopyOptions",
]

81
packetary/api/context.py Normal file
View File

@ -0,0 +1,81 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mirantis, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
from packetary.library.connections import ConnectionsManager
from packetary.library.executor import AsynchronousSection
class Configuration(object):
"""The configuration holder."""
def __init__(self, http_proxy=None, https_proxy=None,
retries_num=0, retry_interval=0, threads_num=0,
ignore_errors_num=0):
"""Initialises.
:param http_proxy: the url of proxy for connections over http,
no-proxy will be used if it is not specified
:param https_proxy: the url of proxy for connections over https,
no-proxy will be used if it is not specified
:param retries_num: the number of retries on errors
:param retry_interval: the minimal time between retries (in seconds)
:param threads_num: the max number of active threads
:param ignore_errors_num: the number of errors that may occurs
before stop processing
"""
self.http_proxy = http_proxy
self.https_proxy = https_proxy
self.ignore_errors_num = ignore_errors_num
self.retries_num = retries_num
self.retry_interval = retry_interval
self.threads_num = threads_num
class Context(object):
"""The infra-objects holder."""
def __init__(self, config):
"""Initialises.
:param config: the configuration
"""
self._connection = ConnectionsManager(
proxy=config.http_proxy,
secure_proxy=config.https_proxy,
retries_num=config.retries_num,
retry_interval=config.retry_interval
)
self._threads_num = config.threads_num
self._ignore_errors_num = config.ignore_errors_num
@property
def connection(self):
"""Gets the connection."""
return self._connection
def async_section(self, ignore_errors_num=None):
"""Gets the execution scope.
:param ignore_errors_num: custom value for ignore_errors_num,
the class value is used if omitted.
"""
if ignore_errors_num is None:
ignore_errors_num = self._ignore_errors_num
return AsynchronousSection(self._threads_num, ignore_errors_num)

108
packetary/api/loaders.py Normal file
View File

@ -0,0 +1,108 @@
# -*- coding: utf-8 -*-
# Copyright 2016 Mirantis, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import re
import six
from packetary.objects.package_relation import PackageRelation
def make_pattern_match(name, value):
return lambda o: re.match(value, getattr(o, name))
def make_exact_match(name, value):
return lambda o: getattr(o, name) == value
def make_attr_match(name, value):
if value.startswith('/') and value.endswith('/'):
return make_pattern_match(name, value[1:-1])
return make_exact_match(name, value)
def all_of(operators):
return lambda o: all((x(o) for x in operators))
def any_of(operators):
return lambda o: any((f(o) for f in operators))
def load_filters(data):
"""Loads filter from filter data.
Property value could be a string or a python regexp.
Example of filters data:
- name: full-package-name
section: section1
- name: /^.*substr/
:param data: A list of filters
:return: Lambda that could match a particular package.
"""
return any_of([
all_of([make_attr_match(n, v) for n, v in six.iteritems(attrs)])
for attrs in data
])
def load_package_relations(data, consumer):
"""Gets the list PackageRelations from descriptions.
:param data: the descriptions of package relations
:param consumer: the result consumer
"""
if not data:
return
for d in data:
versions = d.get('versions', None)
if versions is None:
consumer(PackageRelation.from_args((d['name'],)))
else:
for version in versions:
consumer(PackageRelation.from_args(
([d['name']] + version.split(None, 1))
))
def get_packages_traverse(data, consumer):
"""Gets the traverse to get all packages from repository as relations.
:param data: the description of repositories to traverse
:param consumer: the requirements consumer
:return: callable that expects package as argument
"""
if not data:
return lambda _: None
filters_per_repo = {
d['name']: load_filters(d.get('excludes', ()))
for d in data
}
def traverse(pkg):
if pkg.repository.name in filters_per_repo:
excludes_filter = filters_per_repo[pkg.repository.name]
if not excludes_filter(pkg):
consumer(
PackageRelation.from_args((pkg.name, '=', pkg.version))
)
return traverse

29
packetary/schemas/packages_schema.py → packetary/api/options.py
View File

@ -16,27 +16,8 @@
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
PACKAGES_SCHEMA = {
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "array",
"items": {
"type": "object",
"required": [
"name"
],
"properties": {
"name": {
"type": "string"
},
"versions": {
"type": "array",
"items": [
{
"type": "string",
"pattern": "^([<>]=?|=)\s+.+$"
}
]
}
}
}
}
class RepositoryCopyOptions(object):
def __init__(self, sources=False, localizations=False):
self.sources = sources
self.localizations = localizations

166
packetary/api/repositories.py Normal file
View File

@ -0,0 +1,166 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mirantis, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
from collections import defaultdict
import logging
import six
from packetary.api.context import Context
from packetary.api.options import RepositoryCopyOptions
from packetary.controllers import RepositoryController
from packetary.library.functions import compose
from packetary import objects
from packetary import schemas
from packetary.api.loaders import get_packages_traverse
from packetary.api.loaders import load_package_relations
from packetary.api.statistics import CopyStatistics
from packetary.api.validators import declare_schema
logger = logging.getLogger(__package__)
class RepositoryApi(object):
"""Provides high-level API to operate with repositories."""
CopyOptions = RepositoryCopyOptions
def __init__(self, controller):
"""Initialises.
:param controller: the repository controller.
"""
self.controller = controller
def _get_repository_data_schema(self):
return self.controller.get_repository_data_schema()
def _get_repositories_data_schema(self):
return {
'$schema': 'http://json-schema.org/draft-04/schema#',
'type': 'array',
'items': self._get_repository_data_schema()
}
@classmethod
def create(cls, config, repotype, repoarch):
"""Creates the repository API instance.
:param config: the configuration
:param repotype: the kind of repository(deb, yum, etc)
:param repoarch: the architecture of repository (x86_64 or i386)
"""
context = config if isinstance(config, Context) else Context(config)
return cls(RepositoryController.load(context, repotype, repoarch))
@declare_schema(repo_data=_get_repository_data_schema,
package_files=schemas.PACKAGE_FILES_SCHEMA)
def create_repository(self, repo_data, package_files):
"""Create new repository with specified packages.
:param repo_data: The description of repository
:param package_files: The list of URLs of packages
"""
return self.controller.create_repository(repo_data, package_files)
@declare_schema(repos_data=_get_repositories_data_schema,
requirements_data=schemas.REQUIREMENTS_SCHEMA)
def get_packages(self, repos_data, requirements_data=None):
"""Gets the list of packages from repository(es).
:param repos_data: The list of repository descriptions
:param requirements_data: The list of package`s requirements
that should be included
:return: the set of packages
"""
repositories = self.controller.load_repositories(repos_data)
return self._get_packages(repositories, requirements_data)
@declare_schema(repos_data=_get_repositories_data_schema,
requirements_data=schemas.REQUIREMENTS_SCHEMA)
def clone_repositories(self, repos_data, destination,
requirements_data=None, options=None):
"""Creates the clones of specified repositories in local folder.
:param repos_data: The list of repository descriptions
:param requirements_data: The list of package`s requirements
that should be included
:param destination: the destination folder path
:param options: the repository copy options
:return: count of copied and total packages.
"""
repositories = self.controller.load_repositories(repos_data)
all_packages = self._get_packages(repositories, requirements_data)
package_groups = defaultdict(set)
for pkg in all_packages:
package_groups[pkg.repository].add(pkg)
stat = CopyStatistics()
mirrors = defaultdict(set)
options = options or self.CopyOptions()
# group packages by mirror
for repo, packages in six.iteritems(package_groups):
m = self.controller.fork_repository(repo, destination, options)
mirrors[m].update(packages)
# add new packages to mirrors
for m, pkgs in six.iteritems(mirrors):
self.controller.assign_packages(m, pkgs, stat.on_package_copied)
return stat
@declare_schema(repos_data=_get_repositories_data_schema)
def get_unresolved_dependencies(self, repos_data):
"""Gets list of unresolved dependencies for repository(es).
:param repos_data: The list of repository descriptions
:return: list of unresolved dependencies
"""
packages = objects.PackagesTree()
repositories = self.controller.load_repositories(repos_data)
self._load_packages(repositories, packages.add)
return packages.get_unresolved_dependencies()
def _get_packages(self, repositories, requirements):
if requirements:
forest = objects.PackagesForest()
package_relations = []
load_package_relations(
requirements.get('packages'), package_relations.append
)
packages_traverse = get_packages_traverse(
requirements.get('repositories'), package_relations.append
)
for repo in repositories:
self.controller.load_packages(
repo,
compose(forest.add_tree().add, packages_traverse)
)
return forest.get_packages(
package_relations, requirements.get('mandatory', True)
)
packages = set()
self._load_packages(repositories, packages.add)
return packages
def _load_packages(self, repos, consumer):
for repo in repos:
self.controller.load_packages(repo, consumer)

0
packetary/objects/statistics.py → packetary/api/statistics.py
View File

111
packetary/api/validators.py Normal file
View File

@ -0,0 +1,111 @@
# -*- coding: utf-8 -*-
# Copyright 2016 Mirantis, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import functools
import inspect
import jsonschema
_SENTINEL = object()
def _get_default_arguments(func):
try:
signature = inspect.signature(func)
return {p.name: p.default for p in signature.parameters.values()}
except AttributeError:
pass
args = inspect.getargspec(func)
if args.defaults:
return {
k: v for k, v in zip(reversed(args.args), reversed(args.defaults))
}
return {}
def _get_args_count(func):
try:
return len(inspect.signature(func).parameters)
except AttributeError:
pass
return len(inspect.getargspec(func).args)
def _validate_data(data, schema):
"""Validate the input data using jsonschema validation.
:param data: a data to validate represented as a dict
:param schema: a schema to validate represented as a dict;
must be in JSON Schema Draft 4 format.
"""
try:
jsonschema.validate(data, schema)
except jsonschema.ValidationError as e:
_raise_validation_error("data", e.message, e.path)
except jsonschema.SchemaError as e:
_raise_validation_error("schema", e.message, e.schema_path)
def _raise_validation_error(what, details, path):
message = "Invalid {0}: {1}.".format(what, details)
if path:
message += "\nField: [{0}]".format(
"][".join(repr(p) for p in path)
)
raise ValueError(message)
def _build_validator(schema):
# check that schema is method of class and expected self argument
if callable(schema) and _get_args_count(schema) > 0:
def validator(self, value):
_validate_data(value, schema(self))
elif callable(schema):
def validator(_, value):
_validate_data(value, schema())
else:
def validator(_, value):
return _validate_data(value, schema)
return validator
def declare_schema(**schemas):
"""Declares data schema for function arguments.
:param schemas: the mapping, where key is argument name, value is schema
the schema may be callable object or method of class
in this case the wrapped function should be method of
same class
:raises ValueError: if the passed data does not fit declared schema
"""
def decorator(func):
validators = {k: _build_validator(v) for k, v in schemas.items()}
defaults = _get_default_arguments(func)
@functools.wraps(func)
def wrapper(*args, **kwargs):
bound_args = inspect.getcallargs(func, *args, **kwargs)
for n, v in bound_args.items():
if v is not defaults.get(n, _SENTINEL) and n in validators:
validators[n](args and args[0] or None, v)
return func(*args, **kwargs)
return wrapper
return decorator

30
packetary/cli/commands/base.py
View File

@ -23,7 +23,8 @@ import six
from packetary.cli.commands.utils import make_display_attr_getter
from packetary.cli.commands.utils import read_from_file
from packetary import RepositoryApi
from packetary import api
@six.add_metaclass(abc.ABCMeta)
@ -66,17 +67,17 @@ class BaseRepoCommand(command.Command):
:rtype: object
"""
return self.take_repo_action(
RepositoryApi.create(
api.RepositoryApi.create(
self.app_args, parsed_args.type, parsed_args.arch
),
parsed_args
)
@abc.abstractmethod
def take_repo_action(self, api, parsed_args):
def take_repo_action(self, repo_api, parsed_args):
"""Takes action on repository.
:param api: the RepositoryApi instance
:param repo_api: the RepositoryApi instance
:param parsed_args: the command-line arguments
:return: the action result
"""
@ -104,32 +105,13 @@ class PackagesMixin(object):
def get_parser(self, prog_name):
parser = super(PackagesMixin, self).get_parser(prog_name)
parser.add_argument(
"--skip-mandatory",
dest='include_mandatory',
action='store_false',
default=True,
help="Do not copy mandatory packages."
)
group = parser.add_mutually_exclusive_group()
group.add_argument(
"-p", "--packages",
"-R", "--requirements",
dest='requirements',
type=read_from_file,
metavar='FILENAME',
help="The path to file with list of packages."
"See documentation about format."
)
group.add_argument(
"-f", "--exclude-filter",
dest='exclude_filter_data',
type=read_from_file,
metavar='FILENAME',
help="The path to file with package exclude filter data."
"See documentation about format."
)
return parser

14
packetary/cli/commands/clone.py
View File

@ -47,15 +47,15 @@ class CloneCommand(PackagesMixin, RepositoriesMixin, BaseRepoCommand):
return parser
def take_repo_action(self, api, parsed_args):
stat = api.clone_repositories(
def take_repo_action(self, repo_api, parsed_args):
stat = repo_api.clone_repositories(
parsed_args.repositories,
parsed_args.requirements,
parsed_args.destination,
parsed_args.sources,
parsed_args.locales,
parsed_args.include_mandatory,
filter_data=parsed_args.exclude_filter_data,
parsed_args.requirements,
repo_api.CopyOptions(
sources=parsed_args.sources,
localizations=parsed_args.locales,
)
)
self.stdout.write(
"Packages copied: {0.copied}/{0.total}.\n".format(stat)

4
packetary/cli/commands/create.py
View File

@ -43,8 +43,8 @@ class CreateCommand(BaseRepoCommand):
)
return parser
def take_repo_action(self, api, parsed_args):
api.create_repository(
def take_repo_action(self, repo_api, parsed_args):
repo_api.create_repository(
parsed_args.repository,
parsed_args.package_files
)

11
packetary/cli/commands/packages.py
View File

@ -22,7 +22,8 @@ from packetary.cli.commands.base import RepositoriesMixin
class ListOfPackages(
PackagesMixin, RepositoriesMixin, BaseProduceOutputCommand):
PackagesMixin, RepositoriesMixin, BaseProduceOutputCommand):
"""Gets the list of packages from repository(es)."""
columns = (
@ -37,12 +38,10 @@ class ListOfPackages(
"requires",
)
def take_repo_action(self, api, parsed_args):
return api.get_packages(
def take_repo_action(self, repo_api, parsed_args):
return repo_api.get_packages(
parsed_args.repositories,
parsed_args.requirements,
parsed_args.include_mandatory,
filter_data=parsed_args.exclude_filter_data,
parsed_args.requirements
)

4
packetary/cli/commands/unresolved.py
View File

@ -29,8 +29,8 @@ class ListOfUnresolved(RepositoriesMixin, BaseProduceOutputCommand):
"alternative",
)
def take_repo_action(self, api, parsed_args):
return api.get_unresolved_dependencies(
def take_repo_action(self, repo_api, parsed_args):
return repo_api.get_unresolved_dependencies(
parsed_args.repositories
)

7
packetary/controllers/repository.py
View File

@ -84,13 +84,12 @@ class RepositoryController(object):
connection = self.context.connection
self.driver.get_packages(connection, repository, consumer)
def fork_repository(self, repository, destination, source, locale):
def fork_repository(self, repository, destination, options):
"""Creates copy of repositories.
:param repository: the origin repository
:param destination: the target folder
:param source: If True, the source packages will be copied too.
:param locale: If True, the localisation will be copied too.
:param options: The options, see RepositoryCopyOptions
:return: the mapping origin to cloned repository.
"""
new_path = os.path.join(
@ -101,7 +100,7 @@ class RepositoryController(object):
)
logger.info("cloning repository '%s' to '%s'", repository, new_path)
return self.driver.fork_repository(
self.context.connection, repository, new_path, source, locale
self.context.connection, repository, new_path, options
)
def assign_packages(self, repository, packages, observer=None):

6
packetary/drivers/base.py
View File

@ -55,15 +55,13 @@ class RepositoryDriverBase(object):
"""
@abc.abstractmethod
def fork_repository(self, connection, repository, destination,
source=False, locale=False):
def fork_repository(self, connection, repository, destination, options):
"""Creates the new repository with same metadata.
:param connection: the connection manager instance
:param repository: the source repository
:param destination: the destination folder
:param source: copy source files
:param locale: copy localisation
:param options: the options
:return: The copy of repository
"""

3
packetary/drivers/deb_driver.py
View File

@ -197,8 +197,7 @@ class DebRepositoryDriver(RepositoryDriverBase):
self.logger.info("saved %d packages in %s", count, repository)
self._update_suite_index(repository)
def fork_repository(self, connection, repository, destination,
source=False, locale=False):
def fork_repository(self, connection, repository, destination, options):
# TODO(download gpk)
# TODO(sources and locales)
new_repo = copy.copy(repository)

3
packetary/drivers/rpm_driver.py
View File

@ -162,8 +162,7 @@ class RpmRepositoryDriver(RepositoryDriverBase):
groupstree = self._load_groups(connection, repository)
self._rebuild_repository(connection, repository, packages, groupstree)
def fork_repository(self, connection, repository, destination,
source=False, locale=False):
def fork_repository(self, connection, repository, destination, options):
# TODO(download gpk)
# TODO(sources and locales)
new_repo = copy.copy(repository)

26
packetary/library/functions.py Normal file
View File

@ -0,0 +1,26 @@
# -*- coding: utf-8 -*-
# Copyright 2016 Mirantis, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
def compose(*functions):
"""Call all functions with same arguments."""
def wrapper(*args, **kwargs):
for f in functions:
f(*args, **kwargs)
return wrapper

6
packetary/schemas/__init__.py
View File

@ -18,14 +18,12 @@
from packetary.schemas.deb_repo_schema import DEB_REPO_SCHEMA
from packetary.schemas.package_files_schema import PACKAGE_FILES_SCHEMA
from packetary.schemas.package_filter_schema import PACKAGE_FILTER_SCHEMA
from packetary.schemas.packages_schema import PACKAGES_SCHEMA
from packetary.schemas.requirements_schema import REQUIREMENTS_SCHEMA
from packetary.schemas.rpm_repo_schema import RPM_REPO_SCHEMA
__all__ = [
"DEB_REPO_SCHEMA",
"PACKAGE_FILES_SCHEMA",
"PACKAGE_FILTER_SCHEMA",
"PACKAGES_SCHEMA",
"REQUIREMENTS_SCHEMA",
"RPM_REPO_SCHEMA",
]

74
packetary/schemas/requirements_schema.py Normal file
View File

@ -0,0 +1,74 @@
# -*- coding: utf-8 -*-
# Copyright 2016 Mirantis, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
REQUIREMENTS_SCHEMA = {
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"anyOf": [
{"required": ["packages"]},
{"required": ["repositories"]},
{"required": ["mandatory"]}
],
"properties": {
"repositories": {
"type": "array",
"items": {
"type": "object",
"required": ["name"],
"properties": {
"name": {
"type": "string"
},
"excludes": {
"type": "array",
"items": {
"type": "object",
"patternProperties": {
r"[a-z][\w_]*": {
"type": "string"
}
}
}
}
}
}
},
"packages": {
"type": "array",
"items": {
"type": "object",
"required": ["name"],
"properties": {
"name": {
"type": "string"
},
"versions": {
"type": "array",
"items": {
"type": "string",
"pattern": "^([<>]=?|=)\s+.+$"
}
}
}
}
},
"mandatory": {
"type": "boolean"
},
}
}

2
packetary/tests/base.py
View File

@ -26,6 +26,8 @@ class TestCase(unittest.TestCase):
"""Test case base class for all unit tests."""
maxDiff = None
def _check_cases(self, assertion, cases, method):
for exp, value in cases:
assertion(

6
packetary/tests/stubs/helpers.py
View File

@ -36,11 +36,9 @@ class CallbacksAdapter(mock.MagicMock):
else:
callback = None
if not callable(callback):
return super(CallbacksAdapter, self).__call__(*args, **kwargs)
args = args[:-1]
data = super(CallbacksAdapter, self).__call__(*args, **kwargs)
if not callable(callback):
return data
if isinstance(data, list):
for d in data:

57
packetary/tests/test_api_context.py Normal file
View File

@ -0,0 +1,57 @@
# -*- coding: utf-8 -*-
# Copyright 2016 Mirantis, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import mock
from packetary.api import context
from packetary.tests import base
class TestContext(base.TestCase):
@classmethod
def setUpClass(cls):
cls.config = context.Configuration(
threads_num=2,
ignore_errors_num=3,
retries_num=5,
retry_interval=10,
http_proxy="http://localhost",
https_proxy="https://localhost"
)
@mock.patch("packetary.api.context.ConnectionsManager")
def test_initialise_connection_manager(self, conn_manager):
ctx = context.Context(self.config)
conn_manager.assert_called_once_with(
proxy="http://localhost",
secure_proxy="https://localhost",
retries_num=5,
retry_interval=10
)
self.assertIs(conn_manager(), ctx.connection)
@mock.patch("packetary.api.context.AsynchronousSection")
def test_asynchronous_section(self, async_section):
ctx = context.Context(self.config)
s = ctx.async_section()
async_section.assert_called_with(2, 3)
self.assertIs(s, async_section())
ctx.async_section(0)
async_section.assert_called_with(2, 0)

84
packetary/tests/test_api_loaders.py Normal file
View File

@ -0,0 +1,84 @@
# -*- coding: utf-8 -*-
# Copyright 2016 Mirantis, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
from packetary.api import loaders
from packetary.tests import base
from packetary.tests.stubs import generator
class TestDataLoaders(base.TestCase):
def test_load_filter(self):
filter_data = [
{"name": "p1", "group": "g1"},
{"name": "p2"},
{"group": "g3"},
{"name": "/^.5/", "group": "/^.*3/"},
{"group": "/^.*4/"},
]
filters = loaders.load_filters(filter_data)
cases = [
(True, (generator.gen_package(name='p1', group='g1'),)),
(True, (generator.gen_package(name="p2", group="g1"),)),
(False, (generator.gen_package(name="p3", group="g2"),)),
(True, (generator.gen_package(name="p4", group="g3"),)),
(True, (generator.gen_package(name="p5", group="g3"),)),
(True, (generator.gen_package(name="p6", group="g4"),)),
]
self._check_cases(self.assertIs, cases, filters)
self.assertFalse(loaders.load_filters([])(cases[0][1][0]))
def test_load_package_relations(self):
data = [
{'name': 'test1'},
{'name': 'test2', 'versions': ['> 1', '< 3']},
]
expected = [
str(generator.gen_relation('test1')),
str(generator.gen_relation('test2', ['<', '3'])),
str(generator.gen_relation('test2', ['>', '1'])),
]
actual = []
loaders.load_package_relations(data, lambda x: actual.append(str(x)))
self.assertItemsEqual(expected, actual)
actual = []
loaders.load_package_relations(None, actual.append)
self.assertEqual([], actual)
def test_get_packages_traverse(self):
data = [{
'name': 'r1',
'excludes': [{'name': 'p1'}]
}]
repo = generator.gen_repository(name='r1')
repo2 = generator.gen_repository(name='r2')
packages = [
generator.gen_package(name='p1', version=1, repository=repo),
generator.gen_package(name='p2', version=2, repository=repo),
generator.gen_package(name='p3', version=2, repository=repo2),
generator.gen_package(name='p4', version=2, repository=repo2)
]
actual = []
traverse = loaders.get_packages_traverse(
data, lambda x: actual.append(str(x))
)
for p in packages:
traverse(p)
expected = [str(generator.gen_relation('p2', ['=', '2']))]
self.assertItemsEqual(expected, actual)

91
packetary/tests/test_api_validators.py Normal file
View File

@ -0,0 +1,91 @@
# -*- coding: utf-8 -*-
# Copyright 2016 Mirantis, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import mock
from jsonschema import SchemaError
from jsonschema import ValidationError
from packetary.api import validators
from packetary.tests import base
@mock.patch('packetary.api.validators.jsonschema',
ValidationError=ValidationError, SchemaError=SchemaError)
class TestDataValidators(base.TestCase):
@classmethod
def setUpClass(cls):
cls.data = {'key': 'value'}
cls.schema = {'type': 'object'}
def test_validate_data(self, jsonschema_mock):
validators._validate_data(self.data, self.schema)
jsonschema_mock.validate.assert_called_once_with(
self.data, self.schema
)
def test_validate_invalid_data(self, jsonschema_mock):
paths = [(("a", 0), "\['a'\]\[0\]"), ((), "")]
for path, details in paths:
msg = "Invalid data: error."
if details:
msg += "\nField: {0}".format(details)
with self.assertRaisesRegexp(ValueError, msg):
jsonschema_mock.validate.side_effect = ValidationError(
"error", path=path
)
validators._validate_data(self.data, self.schema)
msg = "Invalid schema: error."
if details:
msg += "\nField: {0}".format(details)
with self.assertRaisesRegexp(ValueError, msg):
jsonschema_mock.validate.side_effect = SchemaError(
"error", schema_path=path
)
validators._validate_data(self.data, self.schema)
def test_build_validator(self, jsonschema_mock):
schemas = [
lambda this: this.schema,
lambda: self.schema,
self.schema
]
for schema in schemas:
jsonschema_mock.reset()
validator = validators._build_validator(schema)
validator(self, self.data)
jsonschema_mock.validate.assert_called_with(self.data, self.schema)
def test_declare_schema_default_does_not_check(self, jsonschema_mock):
func = validators.declare_schema(p=self.schema)(lambda p=None: None)
func(None)
self.assertEqual(0, jsonschema_mock.validate.call_count)
def test_declare_schema_check_data(self, jsonschema_mock):
func = validators.declare_schema(p=self.schema)(lambda p: None)
func(self.data)
jsonschema_mock.validate.assert_called_with(self.data, self.schema)
def test_declare_schema_if_schema_is_method(self, jsonschema_mock):
func = validators.declare_schema(p=lambda x: x.schema)(
lambda x, p: None
)
func(self, self.data)
jsonschema_mock.validate.assert_called_with(self.data, self.schema)

27
packetary/tests/test_cli_commands.py
View File

@ -16,20 +16,22 @@
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import mock
import subprocess
import mock
# The cmd2 does not work with python3.5
# because it tries to get access to the property mswindows,
# that was removed in 3.5
subprocess.mswindows = False
from packetary.api import RepositoryApi
from packetary.api.statistics import CopyStatistics
from packetary.cli.commands import clone
from packetary.cli.commands import create
from packetary.cli.commands import packages
from packetary.cli.commands import unresolved
from packetary.objects.statistics import CopyStatistics
from packetary.tests import base
from packetary.tests.stubs.generator import gen_package
from packetary.tests.stubs.generator import gen_relation
@ -38,7 +40,7 @@ from packetary.tests.stubs.generator import gen_repository
@mock.patch("packetary.cli.commands.base.BaseRepoCommand.stdout")
@mock.patch("packetary.cli.commands.base.read_from_file")
@mock.patch("packetary.cli.commands.base.RepositoryApi")
@mock.patch("packetary.cli.commands.base.api.RepositoryApi")
class TestCliCommands(base.TestCase):
common_argv = [
"--ignore-errors-num=3",
@ -51,11 +53,10 @@ class TestCliCommands(base.TestCase):
clone_argv = [
"-r", "repositories.yaml",
"-p", "packages.yaml",
"-R", "requirements.yaml",
"-d", "/root",
"-t", "deb",
"-a", "x86_64",
"--skip-mandatory"
]
create_argv = [
@ -95,7 +96,7 @@ class TestCliCommands(base.TestCase):
def test_clone_cmd(self, api_mock, read_file_mock, stdout_mock):
read_file_mock.side_effect = [
[{"name": "repo"}],
[{"name": "package"}],
{'packages': [{"name": "package"}]},
]
api_instance = self.get_api_instance_mock(api_mock)
api_instance.clone_repositories.return_value = CopyStatistics()
@ -105,13 +106,13 @@ class TestCliCommands(base.TestCase):
)
self.check_common_config(api_mock.create.call_args[0][0])
read_file_mock.assert_any_call("repositories.yaml")
read_file_mock.assert_any_call("packages.yaml")
read_file_mock.assert_any_call("requirements.yaml")
api_instance.clone_repositories.assert_called_once_with(
[{"name": "repo"}], [{"name": "package"}], "/root",
False,
False,
False,
filter_data=None,
[{"name": "repo"}], "/root", {'packages': [{"name": "package"}]},
api_instance.CopyOptions.return_value
)
api_instance.CopyOptions.assert_called_once_with(
sources=False, localizations=False,
)
stdout_mock.write.assert_called_once_with(
"Packages copied: 0/0.\n"
@ -132,7 +133,7 @@ class TestCliCommands(base.TestCase):
)
self.check_common_config(api_mock.create.call_args[0][0])
api_instance.get_packages.assert_called_once_with(
[{"name": "repo"}], None, True, filter_data=None
[{"name": "repo"}], None
)
self.assertIn(
"test1; test1.pkg",

2
packetary/tests/test_deb_driver.py
View File

@ -230,7 +230,7 @@ class TestDebDriver(base.TestCase):
]
open.side_effect = files
new_repo = self.driver.fork_repository(
self.connection, self.repo, "/root/test"
self.connection, self.repo, "/root/test", None
)
self.assertEqual(self.repo.name, new_repo.name)
self.assertEqual(self.repo.architecture, new_repo.architecture)

448
packetary/tests/test_repository_api.py
View File

@ -19,62 +19,77 @@
import copy
import mock
import jsonschema
from packetary import api
from packetary.controllers.repository import RepositoryController
from packetary import schemas
from packetary.api import Configuration
from packetary.api import Context
from packetary.api import RepositoryApi
from packetary.schemas import PACKAGE_FILES_SCHEMA
from packetary.schemas import PACKAGE_FILTER_SCHEMA
from packetary.schemas import PACKAGES_SCHEMA
from packetary.tests import base
from packetary.tests.stubs import generator
from packetary.tests.stubs.helpers import CallbacksAdapter
@mock.patch("packetary.api.jsonschema")
@mock.patch("packetary.api.validators.jsonschema")
class TestRepositoryApi(base.TestCase):
def setUp(self):
self.controller = CallbacksAdapter()
self.api = RepositoryApi(self.controller)
self.repo_data = {"name": "repo1", "uri": "file:///repo1"}
self.requirements_data = [
{"name": "test1"}, {"name": "test2", "versions": ["< 3", "> 1"]}
]
self.controller = CallbacksAdapter(spec=RepositoryController)
self.api = api.RepositoryApi(self.controller)
self.schema = {}
self.repo = generator.gen_repository(**self.repo_data)
self.controller.load_repositories.return_value = [self.repo]
self.controller.get_repository_data_schema.return_value = self.schema
self._generate_packages()
def _generate_repositories(self, count=1):
self.repos_data = [
{"name": "repo{0}".format(i), "uri": "file:///repo{0}".format(i)}
for i in range(count)
]
self.repos = [
generator.gen_repository(**data) for data in self.repos_data
]
self.controller.load_repositories.return_value = self.repos
def _generate_mirrors(self):
mirrors = {}
for repo in self.repos:
mirror = copy.copy(repo)
mirror.url = "file:///mirror/{0}".format(repo.name)
mirrors[repo] = mirror
self.controller.fork_repository.side_effect = lambda *x: mirrors[x[0]]
self.mirrors = mirrors
def _generate_packages(self):
self.packages = [
generator.gen_package(idx=1, repository=self.repo, requires=None),
generator.gen_package(idx=2, repository=self.repo, requires=None),
generator.gen_package(
idx=3, repository=self.repo, mandatory=True,
requires=[generator.gen_relation("package2")]
),
generator.gen_package(
idx=4, repository=self.repo, mandatory=False,
requires=[generator.gen_relation("package1")]
),
generator.gen_package(
idx=5, repository=self.repo,
requires=[generator.gen_relation("package6")])
[
generator.gen_package(
name='{0}_1'.format(r.name), repository=r, requires=None
),
generator.gen_package(
name='{0}_2'.format(r.name), repository=r, requires=None
),
generator.gen_package(
name='{0}_3'.format(r.name), repository=r, mandatory=True,
requires=[generator.gen_relation("{0}_2".format(r.name))]
),
generator.gen_package(
name='{0}_4'.format(r.name), repository=r, mandatory=False,
requires=[generator.gen_relation("{0}_1".format(r.name))]
),
generator.gen_package(
name='{0}_5'.format(r.name), repository=r,
requires=[generator.gen_relation("unresolved")]
),
]
for r in self.repos
]
self.controller.load_packages.return_value = self.packages
self.controller.load_packages.side_effect = self.packages
@mock.patch("packetary.api.RepositoryController")
@mock.patch("packetary.api.ConnectionsManager")
def test_create_with_config(self, connection_mock, controller_mock,
jsonschema_mock):
config = Configuration(
@mock.patch("packetary.api.context.ConnectionsManager")
@mock.patch("packetary.api.repositories.RepositoryController")
def test_create_with_config(self, controller_mock, connection_mock, _):
config = api.Configuration(
http_proxy="http://localhost", https_proxy="https://localhost",
retries_num=10, retry_interval=1, threads_num=8,
ignore_errors_num=6
)
RepositoryApi.create(config, "deb", "x86_64")
api.RepositoryApi.create(config, "deb", "x86_64")
connection_mock.assert_called_once_with(
proxy="http://localhost",
secure_proxy="https://localhost",
@ -85,17 +100,16 @@ class TestRepositoryApi(base.TestCase):
mock.ANY, "deb", "x86_64"
)
@mock.patch("packetary.api.RepositoryController")
@mock.patch("packetary.api.ConnectionsManager")
def test_create_with_context(self, connection_mock, controller_mock,
jsonschema_mock):
config = Configuration(
@mock.patch("packetary.api.context.ConnectionsManager")
@mock.patch("packetary.api.repositories.RepositoryController")
def test_create_with_context(self, controller_mock, connection_mock, _):
config = api.Configuration(
http_proxy="http://localhost", https_proxy="https://localhost",
retries_num=10, retry_interval=1, threads_num=8,
ignore_errors_num=6
)
context = Context(config)
RepositoryApi.create(context, "deb", "x86_64")
context = api.Context(config)
api.RepositoryApi.create(context, "deb", "x86_64")
connection_mock.assert_called_once_with(
proxy="http://localhost",
secure_proxy="https://localhost",
@ -108,299 +122,109 @@ class TestRepositoryApi(base.TestCase):
def test_create_repository(self, jsonschema_mock):
file_urls = ["file://test1.pkg"]
self.api.create_repository(self.repo_data, file_urls)
self._generate_repositories(1)
self.api.create_repository(self.repos_data[0], file_urls)
self.controller.create_repository.assert_called_once_with(
self.repo_data, file_urls
)
jsonschema_mock.validate.assert_has_calls(
[
mock.call(self.repo_data, self.schema),
mock.call(file_urls, PACKAGE_FILES_SCHEMA),
]
self.repos_data[0], file_urls
)
jsonschema_mock.validate.assert_has_calls([
mock.call(self.repos_data[0], self.schema),
mock.call(file_urls, schemas.PACKAGE_FILES_SCHEMA),
], any_order=True)
def test_get_packages_as_is(self, jsonschema_mock):
packages = self.api.get_packages([self.repo_data], None, False, None)
self.assertEqual(5, len(packages))
self.assertItemsEqual(
self.packages,
packages
)
self._generate_repositories(1)
self._generate_packages()
packages = self.api.get_packages(self.repos_data)
self.assertEqual(5, len(self.packages[0]))
self.assertItemsEqual(self.packages[0], packages)
jsonschema_mock.validate.assert_called_once_with(
self.repo_data, self.schema
self.repos_data, self.api._get_repositories_data_schema()
)
def test_get_packages_by_requirements_with_mandatory(self,
jsonschema_mock):
requirements = [{"name": "package1"}]
packages = self.api.get_packages(
[self.repo_data], requirements, True, None
)
self.assertEqual(3, len(packages))
def test_get_packages_by_requirements(self, jsonschema_mock):
self._generate_repositories(2)
self._generate_packages()
requirements = {
'packages': [{"name": "repo0_1"}],
'repositories': [{"name": "repo1"}],
'mandatory': True
}
packages = self.api.get_packages(self.repos_data, requirements)
expected_packages = self.packages[0][:3] + self.packages[1]
self.assertItemsEqual(
["package1", "package2", "package3"],
(x.name for x in packages)
)
jsonschema_mock.validate.assert_has_calls(
[
mock.call(self.repo_data, self.schema),
mock.call(requirements, PACKAGES_SCHEMA),
]
)
def test_get_packages_by_requirements_without_mandatory(self,
jsonschema_mock):
requirements = [{"name": "package4"}]
packages = self.api.get_packages(
[self.repo_data], requirements, False, None
)
self.assertEqual(2, len(packages))
self.assertItemsEqual(
["package1", "package4"],
(x.name for x in packages)
)
jsonschema_mock.validate.assert_has_calls(
[
mock.call(self.repo_data, self.schema),
mock.call(requirements, PACKAGES_SCHEMA),
]
[x.name for x in expected_packages],
[x.name for x in packages]
)
repos_schema = self.api._get_repositories_data_schema()
jsonschema_mock.validate.assert_has_calls([
mock.call(self.repos_data, repos_schema),
mock.call(requirements, schemas.REQUIREMENTS_SCHEMA)
], any_order=True)
def test_clone_repositories_as_is(self, jsonschema_mock):
# return value is used as statistics
mirror = copy.copy(self.repo)
mirror.url = "file:///mirror/repo"
self.controller.fork_repository.return_value = mirror
self._generate_repositories(1)
self._generate_packages()
self._generate_mirrors()
self.controller.assign_packages.return_value = [0, 1, 1, 1, 0, 6]
stats = self.api.clone_repositories([self.repo_data], None, "/mirror")
options = api.RepositoryCopyOptions()
stats = self.api.clone_repositories(
self.repos_data, "/mirror", options=options)
self.controller.fork_repository.assert_called_once_with(
self.repo, '/mirror', False, False
self.repos[0], '/mirror', options
)
self.controller.assign_packages.assert_called_once_with(
mirror, set(self.packages)
self.mirrors[self.repos[0]], set(self.packages[0]), mock.ANY
)
self.assertEqual(6, stats.total)
self.assertEqual(4, stats.copied)
jsonschema_mock.validate.assert_called_once_with(
self.repo_data, self.schema
self.repos_data, self.api._get_repositories_data_schema()
)
def test_clone_by_requirements_with_mandatory(self, jsonschema_mock):
# return value is used as statistics
mirror = copy.copy(self.repo)
mirror.url = "file:///mirror/repo"
requirements = [{"name": "package1"}]
self.controller.fork_repository.return_value = mirror
self.controller.assign_packages.return_value = [0, 1, 1]
def test_clone_by_requirements(self, jsonschema_mock):
self._generate_repositories(2)
self._generate_packages()
self._generate_mirrors()
requirements = {
'packages': [{"name": "repo0_1"}],
'repositories': [{"name": "repo1"}],
'mandatory': False
}
self.controller.assign_packages.return_value = [0, 1, 1] * 3
stats = self.api.clone_repositories(
[self.repo_data], requirements,
"/mirror", include_mandatory=True
self.repos_data, "/mirror", requirements
)
packages = {self.packages[0], self.packages[1], self.packages[2]}
self.controller.fork_repository.assert_called_once_with(
self.repo, '/mirror', False, False
self.controller.fork_repository.assert_has_calls(
[mock.call(r, '/mirror', mock.ANY) for r in self.repos],
any_order=True
)
self.controller.assign_packages.assert_called_once_with(
mirror, packages
)
self.assertEqual(3, stats.total)
self.assertEqual(2, stats.copied)
jsonschema_mock.validate.assert_has_calls(
[
mock.call(self.repo_data, self.schema),
mock.call(requirements, PACKAGES_SCHEMA),
]
)
def test_clone_by_requirements_without_mandatory(self,
jsonschema_mock):
# return value is used as statistics
mirror = copy.copy(self.repo)
mirror.url = "file:///mirror/repo"
requirements = [{"name": "package4"}]
self.controller.fork_repository.return_value = mirror
self.controller.assign_packages.return_value = [0, 4]
stats = self.api.clone_repositories(
[self.repo_data], requirements,
"/mirror", include_mandatory=False
)
packages = {self.packages[0], self.packages[3]}
self.controller.fork_repository.assert_called_once_with(
self.repo, '/mirror', False, False
)
self.controller.assign_packages.assert_called_once_with(
mirror, packages
)
self.assertEqual(2, stats.total)
self.assertEqual(1, stats.copied)
jsonschema_mock.validate.assert_has_calls(
[
mock.call(self.repo_data, self.schema),
mock.call(requirements, PACKAGES_SCHEMA),
]
)
def test_clone_with_filter(self, jsonschema_mock):
repos_data = "repos_data"
requirements_data = "requirements_data"
filter_data = "filter_data"
repos = "repos"
requirements = "requirements"
exclude_filter = "exclude_filter"
self.api._load_repositories = mock.Mock(return_value=repos)
self.api._load_requirements = mock.Mock(return_value=requirements)
self.api._load_filter = mock.Mock(return_value=exclude_filter)
self.api._get_packages = mock.Mock(return_value=set())
self.api.controller = mock.Mock()
self.api.clone_repositories(repos_data, requirements_data,
"destination", filter_data=filter_data)
self.api._load_repositories.assert_called_once_with(repos_data)
self.api._load_requirements.assert_called_once_with(requirements_data)
self.api._load_filter.assert_called_once_with(filter_data)
self.api._get_packages.assert_called_once_with(
repos, requirements, False, exclude_filter)
def test_get_packages_with_exclude_filter(self, jsonschema_mock):
exclude_filter = lambda p: any([p == "p1", p == "p3"])
self.api._load_packages = CallbacksAdapter()
self.api._load_packages.return_value = ["p1", "p2", "p3", "p4"]
packages = self.api._get_packages("repos", None, False, exclude_filter)
self.assertSetEqual(packages, set(["p2", "p4"]))
def test_get_packages_without_exclude_filter(self, jsonschema_mock):
self.api._load_packages = CallbacksAdapter()
self.api._load_packages.return_value = ["p1", "p2"]
packages = self.api._get_packages("repos", None, False, None)
self.assertSetEqual(packages, set(["p1", "p2"]))
self.controller.assign_packages.assert_has_calls([
mock.call(
self.mirrors[self.repos[0]],
set(self.packages[0][:1]),
mock.ANY
),
mock.call(
self.mirrors[self.repos[1]],
set(self.packages[1]),
mock.ANY
)
], any_order=True)
self.assertEqual(18, stats.total)
self.assertEqual(12, stats.copied)
repos_schema = self.api._get_repositories_data_schema()
jsonschema_mock.validate.assert_has_calls([
mock.call(self.repos_data, repos_schema),
mock.call(requirements, schemas.REQUIREMENTS_SCHEMA)
], any_order=True)
def test_get_unresolved(self, jsonschema_mock):
unresolved = self.api.get_unresolved_dependencies([self.repo_data])
self.assertItemsEqual(["package6"], (x.name for x in unresolved))
self._generate_repositories(1)
self._generate_packages()
unresolved = self.api.get_unresolved_dependencies(self.repos_data)
self.assertItemsEqual(["unresolved"], (x.name for x in unresolved))
jsonschema_mock.validate.assert_called_once_with(
self.repo_data, self.schema
self.repos_data, self.api._get_repositories_data_schema()
)
def test_load_filter_with_none(self, jsonschema_mock):
self.assertIsNone(self.api._load_filter(None))
def test_load_filter(self, jsonschema_mock):
self.api._validate_filter_data = mock.Mock()
filter_data = [
{"name": "p1", "group": "g1"},
{"name": "p2"},
{"group": "g3"},
{"name": "/^.5/", "group": "/^.*3/"},
{"group": "/^.*4/"},
]
exclude_filter = self.api._load_filter(filter_data)
p1 = generator.gen_package(name="p1", group="g1")
p2 = generator.gen_package(name="p2", group="g1")
p3 = generator.gen_package(name="p3", group="g2")
p4 = generator.gen_package(name="p4", group="g3")
p5 = generator.gen_package(name="p5", group="g3")
p6 = generator.gen_package(name="p6", group="g4")
cases = [
(True, (p1,)),
(True, (p2,)),
(False, (p3,)),
(True, (p4,)),
(True, (p5,)),
(True, (p6,)),
]
self._check_cases(self.assertEqual, cases, exclude_filter)
def test_validate_filter_data(self, jsonschema_mock):
self.api._validate_data = mock.Mock()
self.api._validate_filter_data("filter_data")
self.api._validate_data.assert_called_once_with("filter_data",
PACKAGE_FILTER_SCHEMA)
def test_load_requirements(self, jsonschema_mock):
expected = {
generator.gen_relation("test1"),
generator.gen_relation("test2", ["<", "3"]),
generator.gen_relation("test2", [">", "1"]),
}
actual = set(self.api._load_requirements(
self.requirements_data
))
self.assertEqual(expected, actual)
self.assertIsNone(self.api._load_requirements(None))
jsonschema_mock.validate.assert_called_once_with(
self.requirements_data,
PACKAGES_SCHEMA
)
def test_validate_data(self, jsonschema_mock):
self.api._validate_data(self.repo_data, self.schema)
jsonschema_mock.validate.assert_called_once_with(
self.repo_data, self.schema
)
def test_validate_invalid_data(self, jschema_m):
jschema_m.ValidationError = jsonschema.ValidationError
jschema_m.SchemaError = jsonschema.SchemaError
paths = [(("a", 0), "\['a'\]\[0\]"), ((), "")]
for path, details in paths:
msg = "Invalid data: error."
if details:
msg += "\nField: {0}".format(details)
with self.assertRaisesRegexp(ValueError, msg):
jschema_m.validate.side_effect = jsonschema.ValidationError(
"error", path=path
)
self.api._validate_data([], {})
jschema_m.validate.assert_called_with([], {})
jschema_m.validate.reset_mock()
msg = "Invalid schema: error."
if details:
msg += "\nField: {0}".format(details)
with self.assertRaisesRegexp(ValueError, msg):
jschema_m.validate.side_effect = jsonschema.SchemaError(
"error", schema_path=path
)
self.api._validate_data([], {})
jschema_m.validate.assert_called_with([], {})
class TestContext(base.TestCase):
@classmethod
def setUpClass(cls):
cls.config = Configuration(
threads_num=2,
ignore_errors_num=3,
retries_num=5,
retry_interval=10,
http_proxy="http://localhost",
https_proxy="https://localhost"
)
@mock.patch("packetary.api.ConnectionsManager")
def test_initialise_connection_manager(self, conn_manager):
context = Context(self.config)
conn_manager.assert_called_once_with(
proxy="http://localhost",
secure_proxy="https://localhost",
retries_num=5,
retry_interval=10
)
self.assertIs(
conn_manager(),
context.connection
)
@mock.patch("packetary.api.AsynchronousSection")
def test_asynchronous_section(self, async_section):
context = Context(self.config)
s = context.async_section()
async_section.assert_called_with(2, 3)
self.assertIs(s, async_section())
context.async_section(0)
async_section.assert_called_with(2, 0)

10
packetary/tests/test_repository_contoller.py
View File

@ -60,7 +60,7 @@ class TestRepositoryController(base.TestCase):
repos = self.ctrl.load_repositories([repo_data])
self.driver.get_repository.assert_called_once_with(
self.context.connection, repo_data, self.ctrl.arch
self.context.connection, repo_data, self.ctrl.arch, mock.ANY
)
self.assertEqual([repo], repos)
@ -93,14 +93,14 @@ class TestRepositoryController(base.TestCase):
clone.url = "/root/repo"
self.driver.fork_repository.return_value = clone
self.context.connection.retrieve.side_effect = [0, 10]
self.ctrl.fork_repository(repo, "./repo", False, False)
self.ctrl.fork_repository(repo, "./repo", None)
self.driver.fork_repository.assert_called_once_with(
self.context.connection, repo, "./repo/test", False, False
self.context.connection, repo, "./repo/test", None
)
repo.path = "os"
self.ctrl.fork_repository(repo, "./repo", False, False)
self.ctrl.fork_repository(repo, "./repo", None)
self.driver.fork_repository.assert_called_with(
self.context.connection, repo, "./repo/os", False, False
self.context.connection, repo, "./repo/os", None
)
def test_copy_packages(self):

6
packetary/tests/test_rpm_driver.py
View File

@ -265,7 +265,8 @@ class TestRpmDriver(base.TestCase):
new_repo = self.driver.fork_repository(
self.connection,
repo,
"/repo/os/x86_64"
"/repo/os/x86_64",
None
)
m_ensure_dir_exists.assert_called_once_with("/repo/os/x86_64")
self.assertEqual(repo.name, new_repo.name)
@ -310,7 +311,8 @@ class TestRpmDriver(base.TestCase):
self.driver.fork_repository(
self.connection,
repo,
"/repo/os/x86_64"
"/repo/os/x86_64",
None
)
self.assertEqual(tmp_file.name, md_config.groupfile)
os_mock.unlink.assert_called_once_with(tmp_file.name)

93
packetary/tests/test_schemas.py
View File

@ -18,14 +18,14 @@
import jsonschema
from packetary.schemas import DEB_REPO_SCHEMA
from packetary.schemas import PACKAGE_FILES_SCHEMA
from packetary.schemas import PACKAGES_SCHEMA
from packetary.schemas import RPM_REPO_SCHEMA
from packetary import schemas
from packetary.tests import base
class TestRepositorySchemaBase(base.TestCase):
schema = None
def check_invalid_name(self):
self._check_invalid_type('name')
@ -78,7 +78,7 @@ class TestRepositorySchemaBase(base.TestCase):
class TestDebRepoSchema(TestRepositorySchemaBase):
def setUp(self):
self.schema = DEB_REPO_SCHEMA
self.schema = schemas.DEB_REPO_SCHEMA
def test_valid_repo_data(self):
repo_data = {
@ -142,7 +142,7 @@ class TestDebRepoSchema(TestRepositorySchemaBase):
class TestRpmRepoSchema(TestRepositorySchemaBase):
def setUp(self):
self.schema = RPM_REPO_SCHEMA
self.schema = schemas.RPM_REPO_SCHEMA
def test_valid_repo_data(self):
repo_data = {
@ -170,63 +170,45 @@ class TestRpmRepoSchema(TestRepositorySchemaBase):
self.check_invalid_path()
class TestPackagesSchema(base.TestCase):
class TestRequirementsSchema(base.TestCase):
def setUp(self):
self.schema = PACKAGES_SCHEMA
self.schema = schemas.REQUIREMENTS_SCHEMA
def test_valid_requirements_data(self):
requirements_data = [
{"name": "test1", "versions": [">= 1.1.2", "<= 3"]},
{"name": "test2", "versions": ["< 3", "> 1", ">= 4"]},
{"name": "test3", "versions": ["= 3"]},
{"name": "test4", "versions": ["= 3"]},
{"name": "test4"}
]
requirements_data = {
"packages": [
{"name": "test1", "versions": [">= 1.1.2", "<= 3"]},
{"name": "test2", "versions": ["< 3", "> 1", ">= 4"]},
{"name": "test3", "versions": ["= 3"]},
{"name": "test4", "versions": ["= 3"]},
{"name": "test4"}
],
"repositories": [
{"name": "repo1", "excludes": [{"name": "/a.+/"}]},
{"name": "repo1", "excludes": [{"group": "debug"}]},
{"name": "repo1"}
],
"all_mandatory": True,
"options": ["localizations", "sources"]
}
self.assertNotRaises(
jsonschema.ValidationError, jsonschema.validate, requirements_data,
self.schema
)
def test_validation_fail_for_required_properties(self):
requirements_data = [
[{"versions": ["< 3", "> 1"]}]
def test_validation_fail_if_missed_required_properties(self):
test_data = [
{},
{"packages": [{"version": "=2.0.0"}]},
{"repositories": [{"excludes": [{"name": "test"}]}]},
]
for data in requirements_data:
self.assertRaisesRegexp(
jsonschema.ValidationError,
"is a required property",
jsonschema.validate, data, self.schema
for data in test_data:
self.assertRaises(
jsonschema.ValidationError, jsonschema.validate,
data, self.schema
)
def test_validation_fail_if_name_is_invalid(self):
requirements_data = [
{"name": 123, "versions": [">= 1.1.2", "<= 3"]},
]
self.assertRaisesRegexp(
jsonschema.ValidationError, "123 is not of type 'string'",
jsonschema.validate, requirements_data, self.schema
)
def test_validation_fail_if_versions_not_array(self):
requirements_data = [
{"name": "test1", "versions": 123}
]
self.assertRaisesRegexp(
jsonschema.ValidationError, "123 is not of type 'array'",
jsonschema.validate, requirements_data,
self.schema
)
def test_validation_fail_if_versions_not_string(self):
requirements_data = [
{"name": "test1", "versions": [123]}
]
self.assertRaisesRegexp(
jsonschema.ValidationError, "123 is not of type 'string'",
jsonschema.validate, requirements_data,
self.schema
)
def test_validation_fail_if_versions_not_match(self):
versions = [
["1.1.2"], # relational operator
@ -240,14 +222,15 @@ class TestPackagesSchema(base.TestCase):
for version in versions:
self.assertRaisesRegexp(
jsonschema.ValidationError, "does not match",
jsonschema.validate, version,
self.schema['items']['properties']['versions']
jsonschema.validate,
{"packages": [{"name": "test", "versions": version}]},
self.schema
)
class TestPackageFilesSchema(base.TestCase):
def setUp(self):
self.schema = PACKAGE_FILES_SCHEMA
self.schema = schemas.PACKAGE_FILES_SCHEMA
def test_valid_file_urls(self):
file_urls = [