[packetary] Repository class

class Repository composes from:
* RepositryDriver - low-level support for physical repository. deb, yum, etc.
* RepositoryController - infrastcuture method to communicate with driver
* RepositoryApi - high-level class, that provides methods to work with repository

Change-Id: Iaf868fca982d91089e369d13a6fb381ff879ea73
Implements: blueprint refactor-local-mirror-scripts
Partial-Bug: #1487077
This commit is contained in:
Bulat Gaifullin 2015-10-21 12:58:54 +03:00
parent d3949f3094
commit 671af8e611
15 changed files with 1215 additions and 1 deletions

View File

@ -14,9 +14,18 @@
# License for the specific language governing permissions and limitations
# under the License.
import pbr.version
from packetary.api import Configuration
from packetary.api import Context
from packetary.api import RepositoryApi
__all__ = [
"Configuration",
"Context",
"RepositoryApi",
]
__version__ = pbr.version.VersionInfo(
'packetary').version_string()

217
packetary/api.py Normal file
View File

@ -0,0 +1,217 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import logging
import six
from packetary.controllers import RepositoryController
from packetary.library.connections import ConnectionsManager
from packetary.library.executor import AsynchronousSection
from packetary.objects import Index
from packetary.objects import PackageRelation
from packetary.objects import PackagesTree
from packetary.objects.statistics import CopyStatistics
logger = logging.getLogger(__package__)
class Configuration(object):
"""The configuration holder."""
def __init__(self, http_proxy=None, https_proxy=None,
retries_num=0, threads_num=0,
ignore_errors_num=0):
"""Initialises.
:param http_proxy: the url of proxy for connections over http,
no-proxy will be used if it is not specified
:param https_proxy: the url of proxy for connections over https,
no-proxy will be used if it is not specified
:param retries_num: the number of retries on errors
:param threads_num: the max number of active threads
:param ignore_errors_num: the number of errors that may occurs
before stop processing
"""
self.http_proxy = http_proxy
self.https_proxy = https_proxy
self.ignore_errors_num = ignore_errors_num
self.retries_num = retries_num
self.threads_num = threads_num
class Context(object):
"""The infra-objects holder."""
def __init__(self, config):
"""Initialises.
:param config: the configuration
"""
self._connection = ConnectionsManager(
proxy=config.http_proxy,
secure_proxy=config.https_proxy,
retries_num=config.retries_num
)
self._threads_num = config.threads_num
self._ignore_errors_num = config.ignore_errors_num
@property
def connection(self):
"""Gets the connection."""
return self._connection
def async_section(self, ignore_errors_num=None):
"""Gets the execution scope.
:param ignore_errors_num: custom value for ignore_errors_num,
the class value is used if omitted.
"""
if ignore_errors_num is None:
ignore_errors_num = self._ignore_errors_num
return AsynchronousSection(self._threads_num, ignore_errors_num)
class RepositoryApi(object):
"""Provides high-level API to operate with repositories."""
def __init__(self, controller):
"""Initialises.
:param controller: the repository controller.
"""
self.controller = controller
@classmethod
def create(cls, config, repotype, repoarch):
"""Creates the repository API instance.
:param config: the configuration
:param repotype: the kind of repository(deb, yum, etc)
:param repoarch: the architecture of repository (x86_64 or i386)
"""
context = config if isinstance(config, Context) else Context(config)
return cls(RepositoryController.load(context, repotype, repoarch))
def get_packages(self, origin, debs=None, requirements=None):
"""Gets the list of packages from repository(es).
:param origin: The list of repository`s URLs
:param debs: the list of repository`s URL to calculate list of
dependencies, that will be used to filter packages.
:param requirements: the list of package relations,
to resolve the list of mandatory packages.
:return: the set of packages
"""
repositories = self._get_repositories(origin)
return self._get_packages(repositories, debs, requirements)
def clone_repositories(self, origin, destination, debs=None,
requirements=None, keep_existing=True,
include_source=False, include_locale=False):
"""Creates the clones of specified repositories in local folder.
:param origin: The list of repository`s URLs
:param destination: the destination folder path
:param debs: the list of repository`s URL to calculate list of
dependencies, that will be used to filter packages.
:param requirements: the list of package relations,
to resolve the list of mandatory packages.
:param keep_existing: If False - local packages that does not exist
in original repo will be removed.
:param include_source: if True, the source packages
will be copied as well.
:param include_locale: if True, the locales
will be copied as well.
:return: count of copied and total packages.
"""
repositories = self._get_repositories(origin)
packages = self._get_packages(repositories, debs, requirements)
mirrors = self.controller.clone_repositories(
repositories, destination, include_source, include_locale
)
package_groups = dict((x, set()) for x in repositories)
for pkg in packages:
package_groups[pkg.repository].add(pkg)
stat = CopyStatistics()
for repo, packages in six.iteritems(package_groups):
mirror = mirrors[repo]
logger.info("copy packages from - %s", repo)
self.controller.copy_packages(
mirror, packages, keep_existing, stat.on_package_copied
)
return stat
def get_unresolved_dependencies(self, urls):
"""Gets list of unresolved dependencies for repository(es).
:param urls: The list of repository`s URLs
:return: list of unresolved dependencies
"""
packages = PackagesTree()
self.controller.load_packages(
self._get_repositories(urls),
packages.add
)
return packages.get_unresolved_dependencies()
def _get_repositories(self, urls):
"""Gets the set of repositories by url."""
repositories = set()
self.controller.load_repositories(urls, repositories.add)
return repositories
def _get_packages(self, repositories, master, requirements):
"""Gets the list of packages according to master and requirements."""
if master is None and requirements is None:
packages = set()
self.controller.load_packages(repositories, packages.add)
return packages
packages = PackagesTree()
self.controller.load_packages(repositories, packages.add)
if master is not None:
main_index = Index()
self.controller.load_packages(
self._get_repositories(master),
main_index.add
)
else:
main_index = None
return packages.get_minimal_subset(
main_index,
self._parse_requirements(requirements)
)
@staticmethod
def _parse_requirements(requirements):
"""Gets the list of relations from requirements.
:param requirements: the list of requirement in next format:
'name [cmp version]|[alt [cmp version]]'
"""
if requirements is not None:
return set(
PackageRelation.from_args(
*(x.split() for x in r.split("|"))) for r in requirements
)
return set()

View File

@ -0,0 +1,21 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from packetary.controllers.repository import RepositoryController
__all__ = [
"RepositoryController"
]

View File

@ -0,0 +1,169 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import logging
import os
import six
import stevedore
logger = logging.getLogger(__package__)
urljoin = six.moves.urllib.parse.urljoin
class RepositoryController(object):
"""Implements low-level functionality to communicate with drivers."""
_drivers = None
def __init__(self, context, driver, arch):
self.context = context
self.driver = driver
self.arch = arch
@classmethod
def load(cls, context, driver_name, repoarch):
"""Creates the repository manager.
:param context: the context
:param driver_name: the name of required driver
:param repoarch: the architecture of repository (x86_64 or i386)
"""
if cls._drivers is None:
cls._drivers = stevedore.ExtensionManager(
"packetary.drivers", invoke_on_load=True
)
try:
driver = cls._drivers[driver_name].obj
except KeyError:
raise NotImplementedError(
"The driver {0} is not supported yet.".format(driver_name)
)
return cls(context, driver, repoarch)
def load_repositories(self, urls, consumer):
"""Loads the repository objects from url.
:param urls: the list of repository urls.
:param consumer: the callback to consume objects
"""
if isinstance(urls, six.string_types):
urls = [urls]
connection = self.context.connection
for parsed_url in self.driver.parse_urls(urls):
self.driver.get_repository(
connection, parsed_url, self.arch, consumer
)
def load_packages(self, repositories, consumer):
"""Loads packages from repository.
:param repositories: the repository object
:param consumer: the callback to consume objects
"""
connection = self.context.connection
for r in repositories:
self.driver.get_packages(connection, r, consumer)
def assign_packages(self, repository, packages, keep_existing=True):
"""Assigns new packages to the repository.
It replaces the current repository`s packages.
:param repository: the target repository
:param packages: the set of new packages
:param keep_existing:
if True, all existing packages will be kept as is.
if False, all existing packages, that are not included
to new packages will be removed.
"""
if not isinstance(packages, set):
packages = set(packages)
else:
packages = packages.copy()
if keep_existing:
consume_exist = packages.add
else:
def consume_exist(package):
if package not in packages:
filepath = os.path.join(
package.repository.url, package.filename
)
logger.info("remove package - %s.", filepath)
os.remove(filepath)
self.driver.get_packages(
self.context.connection, repository, consume_exist
)
self.driver.rebuild_repository(repository, packages)
def copy_packages(self, repository, packages, keep_existing, observer):
"""Copies packages to repository.
:param repository: the target repository
:param packages: the set of packages
:param keep_existing: see assign_packages for more details
:param observer: the package copying process observer
"""
with self.context.async_section() as section:
for package in packages:
section.execute(
self._copy_package, repository, package, observer
)
self.assign_packages(repository, packages, keep_existing)
def clone_repositories(self, repositories, destination,
source=False, locale=False):
"""Creates copy of repositories.
:param repositories: the origin repositories
:param destination: the target folder
:param source: If True, the source packages will be copied too.
:param locale: If True, the localisation will be copied too.
:return: the mapping origin to cloned repository.
"""
mirros = dict()
destination = os.path.abspath(destination)
with self.context.async_section(0) as section:
for r in repositories:
section.execute(
self._clone_repository,
r, destination, source, locale, mirros
)
return mirros
def _clone_repository(self, r, destination, source, locale, mirrors):
"""Creates clone of repository and stores it in mirrors."""
clone = self.driver.clone_repository(
self.context.connection, r, destination, source, locale
)
mirrors[r] = clone
def _copy_package(self, target, package, observer):
"""Synchronises remote file to local fs."""
dst_path = os.path.join(target.url, package.filename)
src_path = urljoin(package.repository.url, package.filename)
bytes_copied = self.context.connection.retrieve(
src_path, dst_path, size=package.filesize
)
if package.filesize < 0:
package.filesize = bytes_copied
observer(bytes_copied)

View File

80
packetary/drivers/base.py Normal file
View File

@ -0,0 +1,80 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import abc
import logging
import six
@six.add_metaclass(abc.ABCMeta)
class RepositoryDriverBase(object):
"""The super class for Repository Drivers.
For implementing support of new type of repository:
- inherit this class
- implement all abstract methods
- register implementation in 'packetary.drivers' namespace
"""
def __init__(self):
self.logger = logging.getLogger(__package__)
@abc.abstractmethod
def parse_urls(self, urls):
"""Parses the repository url.
:return: the sequence of parsed urls
"""
@abc.abstractmethod
def get_repository(self, connection, url, arch, consumer):
"""Loads the repository meta information from URL.
:param connection: the connection manager instance
:param url: the repository`s url
:param arch: the repository`s architecture
:param consumer: the callback to consume result
"""
@abc.abstractmethod
def get_packages(self, connection, repository, consumer):
"""Loads packages from repository.
:param connection: the connection manager instance
:param repository: the repository object
:param consumer: the callback to consume result
"""
@abc.abstractmethod
def clone_repository(self, connection, repository, destination,
source=False, locale=False):
"""Creates copy of repository.
:param connection: the connection manager instance
:param repository: the source repository
:param destination: the destination folder
:param source: copy source files
:param locale: copy localisation
:return: The copy of repository
"""
@abc.abstractmethod
def rebuild_repository(self, repository, packages):
"""Re-builds the repository.
:param repository: the target repository
:param packages: the set of packages
"""

View File

@ -20,6 +20,7 @@ from packetary.objects.package import FileChecksum
from packetary.objects.package import Package
from packetary.objects.package_relation import PackageRelation
from packetary.objects.package_relation import VersionRange
from packetary.objects.packages_tree import PackagesTree
from packetary.objects.repository import Repository
@ -28,6 +29,7 @@ __all__ = [
"Index",
"Package",
"PackageRelation",
"PackagesTree",
"Repository",
"VersionRange",
]

View File

@ -0,0 +1,121 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import warnings
from packetary.objects.index import Index
class UnresolvedWarning(UserWarning):
"""Warning about unresolved depends."""
pass
class PackagesTree(Index):
"""Helper class to deal with dependency graph."""
def __init__(self):
super(PackagesTree, self).__init__()
self.mandatory_packages = []
def add(self, package):
super(PackagesTree, self).add(package)
# store all mandatory packages in separated list for quick access
if package.mandatory:
self.mandatory_packages.append(package)
def get_unresolved_dependencies(self, unresolved=None):
"""Gets the set of unresolved dependencies.
:param unresolved: the known list of unresolved packages.
:return: the set of unresolved depends.
"""
return self.__get_unresolved_dependencies(self)
def get_minimal_subset(self, main, requirements):
"""Gets the minimal work subset.
:param main: the main index, to complete requirements.
:param requirements: additional requirements.
:return: The set of resolved depends.
"""
unresolved = set()
resolved = set()
if main is None:
def pkg_filter(*_):
pass
else:
pkg_filter = main.find
self.__get_unresolved_dependencies(main, requirements)
stack = list()
stack.append((None, requirements))
# add all mandatory packages
for pkg in self.mandatory_packages:
stack.append((pkg, pkg.requires))
while len(stack) > 0:
pkg, required = stack.pop()
resolved.add(pkg)
for require in required:
for rel in require:
if rel not in unresolved:
if pkg_filter(rel.name, rel.version) is not None:
break
# use all packages that meets depends
candidates = self.find_all(rel.name, rel.version)
found = False
for cand in candidates:
if cand == pkg:
continue
found = True
if cand not in resolved:
stack.append((cand, cand.requires))
if found:
break
else:
unresolved.add(require)
msg = "Unresolved depends: {0}".format(require)
warnings.warn(UnresolvedWarning(msg))
resolved.remove(None)
return resolved
@staticmethod
def __get_unresolved_dependencies(index, unresolved=None):
"""Gets the set of unresolved dependencies.
:param index: the search index.
:param unresolved: the known list of unresolved packages.
:return: the set of unresolved depends.
"""
if unresolved is None:
unresolved = set()
for pkg in index:
for require in pkg.requires:
for rel in require:
if rel not in unresolved:
candidate = index.find(rel.name, rel.version)
if candidate is not None and candidate != pkg:
break
else:
unresolved.add(require)
return unresolved

View File

@ -0,0 +1,45 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import copy
class CopyStatistics(object):
"""The statistics of packages copying"""
def __init__(self):
# the number of copied packages
self.copied = 0
# the number of total packages
self.total = 0
def on_package_copied(self, bytes_copied):
"""Proceed next copied package."""
if bytes_copied > 0:
self.copied += 1
self.total += 1
def __iadd__(self, other):
if not isinstance(other, CopyStatistics):
raise TypeError
self.copied += other.copied
self.total += other.total
return self
def __add__(self, other):
result = copy.copy(self)
result += other
return result

View File

@ -0,0 +1,27 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
class Executor(object):
def __enter__(self):
return self
def __exit__(self, *_):
return False
@staticmethod
def execute(f, *args, **kwargs):
return f(*args, **kwargs)

View File

@ -0,0 +1,39 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import mock
class CallbacksAdapter(mock.MagicMock):
"""Helper to return data through callback."""
def __call__(self, *args, **kwargs):
if len(args) > 0:
callback = args[-1]
else:
callback = None
if not callable(callback):
return super(CallbacksAdapter, self).__call__(*args, **kwargs)
args = args[:-1]
data = super(CallbacksAdapter, self).__call__(*args, **kwargs)
if isinstance(data, list):
for d in data:
callback(d)
else:
callback(data)

View File

@ -0,0 +1,110 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import warnings
from packetary.objects import Index
from packetary.objects import PackagesTree
from packetary.tests import base
from packetary.tests.stubs import generator
class TestPackagesTree(base.TestCase):
def setUp(self):
super(TestPackagesTree, self).setUp()
def test_get_unresolved_dependencies(self):
ptree = PackagesTree()
ptree.add(generator.gen_package(
1, requires=[generator.gen_relation("unresolved")]))
ptree.add(generator.gen_package(2, requires=None))
ptree.add(generator.gen_package(
3, requires=[generator.gen_relation("package1")]
))
ptree.add(generator.gen_package(
4,
requires=[generator.gen_relation("loop")],
obsoletes=[generator.gen_relation("loop", ["le", 1])]
))
unresolved = ptree.get_unresolved_dependencies()
self.assertItemsEqual(
["loop", "unresolved"],
(x.name for x in unresolved)
)
def test_get_minimal_subset_with_master(self):
ptree = PackagesTree()
ptree.add(generator.gen_package(1, requires=None))
ptree.add(generator.gen_package(2, requires=None))
ptree.add(generator.gen_package(3, requires=None))
ptree.add(generator.gen_package(
4, requires=[generator.gen_relation("package1")]
))
master = Index()
master.add(generator.gen_package(1, requires=None))
master.add(generator.gen_package(
5,
requires=[generator.gen_relation(
"package10",
alternative=generator.gen_relation("package4")
)]
))
unresolved = set([generator.gen_relation("package3")])
resolved = ptree.get_minimal_subset(master, unresolved)
self.assertItemsEqual(
["package3", "package4"],
(x.name for x in resolved)
)
def test_get_minimal_subset_without_master(self):
ptree = PackagesTree()
ptree.add(generator.gen_package(1, requires=None))
ptree.add(generator.gen_package(2, requires=None))
ptree.add(generator.gen_package(
3, requires=[generator.gen_relation("package1")]
))
unresolved = set([generator.gen_relation("package3")])
resolved = ptree.get_minimal_subset(None, unresolved)
self.assertItemsEqual(
["package3", "package1"],
(x.name for x in resolved)
)
def test_mandatory_packages_always_included(self):
ptree = PackagesTree()
ptree.add(generator.gen_package(1, requires=None, mandatory=True))
ptree.add(generator.gen_package(2, requires=None))
ptree.add(generator.gen_package(3, requires=None))
unresolved = set([generator.gen_relation("package3")])
resolved = ptree.get_minimal_subset(None, unresolved)
self.assertItemsEqual(
["package3", "package1"],
(x.name for x in resolved)
)
def test_warning_if_unresolved(self):
ptree = PackagesTree()
ptree.add(generator.gen_package(
1, requires=None))
with warnings.catch_warnings(record=True) as log:
ptree.get_minimal_subset(
None, [generator.gen_relation("package2")]
)
self.assertIn("package2", str(log[0]))

View File

@ -0,0 +1,230 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import mock
from packetary.api import Configuration
from packetary.api import Context
from packetary.api import RepositoryApi
from packetary.tests import base
from packetary.tests.stubs import generator
from packetary.tests.stubs.helpers import CallbacksAdapter
class TestRepositoryApi(base.TestCase):
def test_get_packages_as_is(self):
controller = CallbacksAdapter()
pkg = generator.gen_package(name="test")
controller.load_packages.side_effect = [
pkg
]
api = RepositoryApi(controller)
packages = api.get_packages("file:///repo1")
self.assertEqual(1, len(packages))
package = packages.pop()
self.assertIs(pkg, package)
def test_get_packages_with_depends_resolving(self):
controller = CallbacksAdapter()
controller.load_packages.side_effect = [
[
generator.gen_package(idx=1, requires=None),
generator.gen_package(
idx=2, requires=[generator.gen_relation("package1")]
),
generator.gen_package(
idx=3, requires=[generator.gen_relation("package1")]
),
generator.gen_package(idx=4, requires=None),
generator.gen_package(idx=5, requires=None),
],
generator.gen_package(
idx=6, requires=[generator.gen_relation("package2")]
),
]
api = RepositoryApi(controller)
packages = api.get_packages([
"file:///repo1", "file:///repo2"
],
"file:///repo3", ["package4"]
)
self.assertEqual(3, len(packages))
self.assertItemsEqual(
["package1", "package4", "package2"],
(x.name for x in packages)
)
controller.load_repositories.assert_any_call(
["file:///repo1", "file:///repo2"]
)
controller.load_repositories.assert_any_call(
"file:///repo3"
)
def test_clone_repositories_as_is(self):
controller = CallbacksAdapter()
repo = generator.gen_repository(name="repo1")
packages = [
generator.gen_package(name="test1", repository=repo),
generator.gen_package(name="test2", repository=repo)
]
mirror = generator.gen_repository(name="mirror")
controller.load_repositories.return_value = repo
controller.load_packages.return_value = packages
controller.clone_repositories.return_value = {repo: mirror}
controller.copy_packages.return_value = [0, 1]
api = RepositoryApi(controller)
stats = api.clone_repositories(
["file:///repo1"], "/mirror", keep_existing=True
)
self.assertEqual(2, stats.total)
self.assertEqual(1, stats.copied)
controller.copy_packages.assert_called_once_with(
mirror, set(packages), True
)
def test_copy_minimal_subset_of_repository(self):
controller = CallbacksAdapter()
repo1 = generator.gen_repository(name="repo1")
repo2 = generator.gen_repository(name="repo2")
repo3 = generator.gen_repository(name="repo3")
mirror1 = generator.gen_repository(name="mirror1")
mirror2 = generator.gen_repository(name="mirror2")
pkg_group1 = [
generator.gen_package(
idx=1, requires=None, repository=repo1
),
generator.gen_package(
idx=1, version=2, requires=None, repository=repo1
),
generator.gen_package(
idx=2, requires=None, repository=repo1
)
]
pkg_group2 = [
generator.gen_package(
idx=4,
requires=[generator.gen_relation("package1")],
repository=repo2,
mandatory=True,
)
]
pkg_group3 = [
generator.gen_package(
idx=3, requires=None, repository=repo1
)
]
controller.load_repositories.side_effect = [[repo1, repo2], repo3]
controller.load_packages.side_effect = [
pkg_group1 + pkg_group2 + pkg_group3,
generator.gen_package(
idx=6,
repository=repo3,
requires=[generator.gen_relation("package2")]
)
]
controller.clone_repositories.return_value = {
repo1: mirror1, repo2: mirror2
}
controller.copy_packages.return_value = 1
api = RepositoryApi(controller)
api.clone_repositories(
["file:///repo1", "file:///repo2"], "/mirror",
["file:///repo3"],
keep_existing=True
)
controller.copy_packages.assert_any_call(
mirror1, set(pkg_group1), True
)
controller.copy_packages.assert_any_call(
mirror2, set(pkg_group2), True
)
self.assertEqual(2, controller.copy_packages.call_count)
def test_get_unresolved(self):
controller = CallbacksAdapter()
pkg = generator.gen_package(
name="test", requires=[generator.gen_relation("test2")]
)
controller.load_packages.side_effect = [
pkg
]
api = RepositoryApi(controller)
r = api.get_unresolved_dependencies("file:///repo1")
controller.load_repositories.assert_called_once_with("file:///repo1")
self.assertItemsEqual(
["test2"],
(x.name for x in r)
)
def test_parse_requirements(self):
requirements = RepositoryApi._parse_requirements(
["p1 le 2 | p2 | p3 ge 2"]
)
expected = generator.gen_relation(
"p1",
["le", '2'],
generator.gen_relation(
"p2",
None,
generator.gen_relation(
"p3",
["ge", '2']
)
)
)
self.assertEqual(1, len(requirements))
self.assertEqual(
list(expected),
list(requirements.pop())
)
class TestContext(base.TestCase):
@classmethod
def setUpClass(cls):
cls.config = Configuration(
threads_num=2,
ignore_errors_num=3,
retries_num=5,
http_proxy="http://localhost",
https_proxy="https://localhost"
)
@mock.patch("packetary.api.ConnectionsManager")
def test_initialise_connection_manager(self, conn_manager):
context = Context(self.config)
conn_manager.assert_called_once_with(
proxy="http://localhost",
secure_proxy="https://localhost",
retries_num=5
)
self.assertIs(
conn_manager(),
context.connection
)
@mock.patch("packetary.api.AsynchronousSection")
def test_asynchronous_section(self, async_section):
context = Context(self.config)
s = context.async_section()
async_section.assert_called_with(2, 3)
self.assertIs(s, async_section())
context.async_section(0)
async_section.assert_called_with(2, 0)

View File

@ -0,0 +1,143 @@
# -*- coding: utf-8 -*-
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import copy
import mock
import six
from packetary.controllers import RepositoryController
from packetary.tests import base
from packetary.tests.stubs.executor import Executor
from packetary.tests.stubs.generator import gen_package
from packetary.tests.stubs.generator import gen_repository
from packetary.tests.stubs.helpers import CallbacksAdapter
class TestRepositoryController(base.TestCase):
def setUp(self):
self.driver = mock.MagicMock()
self.context = mock.MagicMock()
self.context.async_section.return_value = Executor()
self.ctrl = RepositoryController(self.context, self.driver, "x86_64")
def test_load_fail_if_unknown_driver(self):
with self.assertRaisesRegexp(NotImplementedError, "unknown_driver"):
RepositoryController.load(
self.context,
"unknown_driver",
"x86_64"
)
@mock.patch("packetary.controllers.repository.stevedore")
def test_load_driver(self, stevedore):
stevedore.ExtensionManager.return_value = {
"test": mock.MagicMock(obj=self.driver)
}
RepositoryController._drivers = None
controller = RepositoryController.load(self.context, "test", "x86_64")
self.assertIs(self.driver, controller.driver)
def test_load_repositories(self):
self.driver.parse_urls.return_value = ["test1"]
consumer = mock.MagicMock()
self.ctrl.load_repositories("file:///test1", consumer)
self.driver.parse_urls.assert_called_once_with(["file:///test1"])
self.driver.get_repository.assert_called_once_with(
self.context.connection, "test1", "x86_64", consumer
)
for url in [six.u("file:///test1"), ["file:///test1"]]:
self.driver.reset_mock()
self.ctrl.load_repositories(url, consumer)
if not isinstance(url, list):
url = [url]
self.driver.parse_urls.assert_called_once_with(url)
def test_load_packages(self):
repo = mock.MagicMock()
consumer = mock.MagicMock()
self.ctrl.load_packages([repo], consumer)
self.driver.get_packages.assert_called_once_with(
self.context.connection, repo, consumer
)
@mock.patch("packetary.controllers.repository.os")
def test_assign_packages(self, os):
repo = gen_repository(url="/test/repo")
packages = [
gen_package(name="test1", repository=repo),
gen_package(name="test2", repository=repo)
]
existed_packages = [
gen_package(name="test3", repository=repo),
gen_package(name="test2", repository=repo)
]
os.path.join = lambda *x: "/".join(x)
self.driver.get_packages = CallbacksAdapter()
self.driver.get_packages.return_value = existed_packages
self.ctrl.assign_packages(repo, packages, True)
os.remove.assert_not_called()
all_packages = set(packages + existed_packages)
self.driver.rebuild_repository.assert_called_once_with(
repo, all_packages
)
self.driver.rebuild_repository.reset_mock()
self.ctrl.assign_packages(repo, packages, False)
self.driver.rebuild_repository.assert_called_once_with(
repo, set(packages)
)
os.remove.assert_called_once_with("/test/repo/test3.pkg")
def test_copy_packages(self):
repo = gen_repository(url="file:///repo/")
packages = [
gen_package(name="test1", repository=repo, filesize=10),
gen_package(name="test2", repository=repo, filesize=-1)
]
target = gen_repository(url="/test/repo")
self.context.connection.retrieve.side_effect = [0, 10]
observer = mock.MagicMock()
self.ctrl.copy_packages(target, packages, True, observer)
observer.assert_has_calls([mock.call(0), mock.call(10)])
self.context.connection.retrieve.assert_any_call(
"file:///repo/test1.pkg",
"/test/repo/test1.pkg",
size=10
)
self.context.connection.retrieve.assert_any_call(
"file:///repo/test2.pkg",
"/test/repo/test2.pkg",
size=-1
)
self.driver.rebuild_repository.assert_called_once_with(
target, set(packages)
)
@mock.patch("packetary.controllers.repository.os")
def test_clone_repository(self, os):
os.path.abspath.return_value = "/root/repo"
repos = [
gen_repository(name="test1"),
gen_repository(name="test2")
]
clones = [copy.copy(x) for x in repos]
self.driver.clone_repository.side_effect = clones
mirrors = self.ctrl.clone_repositories(repos, "./repo")
for r in repos:
self.driver.clone_repository.assert_any_call(
self.context.connection, r, "/root/repo", False, False
)
self.assertEqual(mirrors, dict(zip(repos, clones)))

View File

@ -7,4 +7,5 @@ Babel>=1.3
eventlet>=0.15
bintrees>=2.0.2
chardet>=2.3.0
stevedore>=1.1.0
six>=1.5.2