diff --git a/packetary/library/connections.py b/packetary/library/connections.py index 6d42fd4..97e6c6c 100644 --- a/packetary/library/connections.py +++ b/packetary/library/connections.py @@ -14,12 +14,13 @@ # License for the specific language governing permissions and limitations # under the License. +import errno import logging import os import six import six.moves.http_client as http_client -import six.moves.urllib.request as urllib_request -import six.moves.urllib_error as urllib_error +import six.moves.urllib.request as urllib +import six.moves.urllib_error as urlerror import time from packetary.library.streams import StreamWrapper @@ -31,11 +32,11 @@ logger = logging.getLogger(__package__) RETRYABLE_ERRORS = (http_client.HTTPException, IOError) -class RangeError(urllib_error.URLError): +class RangeError(urlerror.URLError): pass -class RetryableRequest(urllib_request.Request): +class RetryableRequest(urllib.Request): offset = 0 retries_left = 1 start_time = 0 @@ -66,6 +67,7 @@ class ResumableResponse(StreamWrapper): self.request.offset += len(chunk) return chunk except RETRYABLE_ERRORS as e: + # TODO(check hashsums) response = self.opener.error( self.request.get_type(), self.request, self.stream, 502, six.text_type(e), self.stream.info() @@ -73,7 +75,7 @@ class ResumableResponse(StreamWrapper): self.stream = response.stream -class RetryHandler(urllib_request.BaseHandler): +class RetryHandler(urllib.BaseHandler): """urllib Handler to add ability for retrying on server errors.""" @staticmethod @@ -114,23 +116,36 @@ class RetryHandler(urllib_request.BaseHandler): https_response = http_response -class Connection(object): - """Helper class to deal with streams.""" +class ConnectionsManager(object): + """The connections manager.""" - def __init__(self, opener, retries_num): - """Initializes. + def __init__(self, proxy=None, secure_proxy=None, retries_num=0): + """Initialises. - :param opener: the instance of urllib.OpenerDirector + :param proxy: the url of proxy for http-connections + :param secure_proxy: the url of proxy for https-connections :param retries_num: the number of allowed retries """ - self.opener = opener + if proxy: + proxies = { + "http": proxy, + "https": secure_proxy or proxy, + } + else: + proxies = None + self.retries_num = retries_num + self.opener = urllib.build_opener( + RetryHandler(), + urllib.ProxyHandler(proxies) + ) def make_request(self, url, offset=0): """Makes new http request. :param url: the remote file`s url - :param offset: the number of bytes from begin, that will be skipped + :param offset: the number of bytes from the beginning, + that will be skipped :return: The new http request """ @@ -146,14 +161,15 @@ class Connection(object): """Opens remote file for streaming. :param url: the remote file`s url - :param offset: the number of bytes from begin, that will be skipped + :param offset: the number of bytes from the beginning, + that will be skipped """ request = self.make_request(url, offset) while 1: try: return self.opener.open(request) - except (RangeError, urllib_error.HTTPError): + except (RangeError, urlerror.HTTPError): raise except RETRYABLE_ERRORS as e: if request.retries_left <= 0: @@ -169,7 +185,8 @@ class Connection(object): :param url: the remote file`s url :param filename: the file`s name, that includes path on local fs - :param offset: the number of bytes from begin, that will be skipped + :param offset: the number of bytes from the beginning, + that will be skipped """ self._ensure_dir_exists(filename) @@ -180,7 +197,8 @@ class Connection(object): if offset == 0: raise logger.warning( - "Failed to resume download, starts from begin: %s", url + "Failed to resume download, starts from the beginning: %s", + url ) self._copy_stream(fd, url, 0) finally: @@ -194,7 +212,7 @@ class Connection(object): try: os.makedirs(target_dir) except OSError as e: - if e.errno != 17: + if e.errno != errno.EEXIST: raise def _copy_stream(self, fd, url, offset): @@ -202,7 +220,8 @@ class Connection(object): :param fd: the file`s descriptor :param url: the remote file`s url - :param offset: the number of bytes from begin, that will be skipped + :param offset: the number of bytes from the beginning, + that will be skipped """ source = self.open_stream(url, offset) @@ -214,67 +233,3 @@ class Connection(object): if not chunk: break os.write(fd, chunk) - - -class ConnectionContext(object): - """Helper class acquire and release connection within context.""" - def __init__(self, connection, on_exit): - self.connection = connection - self.on_exit = on_exit - - def __enter__(self): - return self.connection - - def __exit__(self, *_): - self.on_exit(self.connection) - - -class ConnectionsPool(object): - """Controls the number of simultaneously opened connections.""" - - MIN_CONNECTIONS_COUNT = 1 - - def __init__(self, count=0, proxy=None, secure_proxy=None, retries_num=0): - """Initialises. - - :param count: the number of allowed simultaneously connections - :param proxy: the url of proxy for http-connections - :param secure_proxy: the url of proxy for https-connections - :param retries_num: the number of allowed retries - """ - if proxy: - proxies = { - "http": proxy, - "https": secure_proxy or proxy, - } - else: - proxies = None - - opener = urllib_request.build_opener( - RetryHandler(), - urllib_request.ProxyHandler(proxies) - ) - - limit = max(count, self.MIN_CONNECTIONS_COUNT) - connections = six.moves.queue.Queue() - while limit > 0: - connections.put(Connection(opener, retries_num)) - limit -= 1 - - self.free = connections - - def get(self, timeout=None): - """Gets the free connection. - - Blocks in case if there is no free connections. - - :param timeout: the timeout in seconds to wait. - by default infinity waiting. - """ - return ConnectionContext( - self.free.get(timeout=timeout), self._release - ) - - def _release(self, connection): - """Puts back connection to free connections.""" - self.free.put(connection) diff --git a/packetary/objects/__init__.py b/packetary/objects/__init__.py new file mode 100644 index 0000000..1c7d351 --- /dev/null +++ b/packetary/objects/__init__.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from packetary.objects.index import Index +from packetary.objects.package import FileChecksum +from packetary.objects.package import Package +from packetary.objects.package_relation import PackageRelation +from packetary.objects.package_relation import VersionRange +from packetary.objects.repository import Repository + + +__all__ = [ + "FileChecksum", + "Index", + "Package", + "PackageRelation", + "Repository", + "VersionRange", +] diff --git a/packetary/objects/base.py b/packetary/objects/base.py new file mode 100644 index 0000000..bfaea47 --- /dev/null +++ b/packetary/objects/base.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import abc +import six + + +@six.add_metaclass(abc.ABCMeta) +class ComparableObject(object): + """Superclass for objects, that should be comparable. + + Note: because python3 does not support __cmp__ slot, use + cmp method to implement all of compare methods. + """ + + @abc.abstractmethod + def cmp(self, other): + """Compares with other object. + + :return: value is negative if if self < other, zero if self == other + strictly positive if x > y + """ + + def __lt__(self, other): + return self.cmp(other) < 0 + + def __le__(self, other): + return self.cmp(other) <= 0 + + def __gt__(self, other): + return self.cmp(other) > 0 + + def __ge__(self, other): + return self.cmp(other) >= 0 + + def __eq__(self, other): + if other is self: + return True + return isinstance(other, type(self)) and self.cmp(other) == 0 + + def __ne__(self, other): + if other is self: + return False + return not isinstance(other, type(self)) or self.cmp(other) != 0 + + def __cmp__(self, other): + return self.cmp(other) diff --git a/packetary/objects/index.py b/packetary/objects/index.py new file mode 100644 index 0000000..8f40094 --- /dev/null +++ b/packetary/objects/index.py @@ -0,0 +1,207 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from bintrees import FastRBTree +from collections import defaultdict +import functools +import operator +import six + + +def _make_operator(direction, op): + """Makes search operator from low-level operation and search direction.""" + return functools.partial(direction, condition=op) + + +def _start_upperbound(versions, version, condition): + """Gets all versions from [start, version] that meet condition. + + :param versions: the tree of versions. + :param version: the required version + :param condition: condition for search + :return: the list of found versions + """ + + result = list(versions.value_slice(None, version)) + try: + bound = versions.ceiling_item(version) + if condition(bound[0], version): + result.append(bound[1]) + except KeyError: + pass + return result + + +def _lowerbound_end(versions, version, condition): + """Gets all versions from [version, end] that meet condition. + + :param versions: the tree of versions. + :param version: the required version + :param condition: condition for search + :return: the list of found versions + """ + result = [] + items = iter(versions.item_slice(version, None)) + bound = next(items, None) + if bound is None: + return result + if condition(bound[0], version): + result.append(bound[1]) + result.extend(x[1] for x in items) + return result + + +def _equal(tree, version): + """Gets the package with specified version.""" + if version in tree: + return [tree[version]] + return [] + + +def _any(tree, _): + """Gets the package with max version.""" + return list(tree.values()) + + +class Index(object): + """The search index for packages. + + Builds three search-indexes: + - index of packages with versions. + - index of virtual packages (provides). + - index of obsoleted packages (obsoletes). + + Uses to find package by name and range of versions. + """ + + operators = { + None: _any, + "lt": _make_operator(_start_upperbound, operator.lt), + "le": _make_operator(_start_upperbound, operator.le), + "gt": _make_operator(_lowerbound_end, operator.gt), + "ge": _make_operator(_lowerbound_end, operator.ge), + "eq": _equal, + } + + def __init__(self): + self.packages = defaultdict(FastRBTree) + self.obsoletes = defaultdict(FastRBTree) + self.provides = defaultdict(FastRBTree) + + def __iter__(self): + """Iterates over all packages including versions.""" + return self.get_all() + + def __len__(self, _reduce=six.functools.reduce): + """Returns the total number of packages with versions.""" + return _reduce( + lambda x, y: x + len(y), + six.itervalues(self.packages), + 0 + ) + + def get_all(self): + """Gets sequence from all of packages including versions.""" + + for versions in six.itervalues(self.packages): + for version in versions.values(): + yield version + + def find(self, name, version): + """Finds the package by name and range of versions. + + :param name: the package`s name. + :param version: the range of versions. + :return: the package if it is found, otherwise None + """ + candidates = self.find_all(name, version) + if len(candidates) > 0: + return candidates[-1] + return None + + def find_all(self, name, version): + """Finds the packages by name and range of versions. + + :param name: the package`s name. + :param version: the range of versions. + :return: the list of suitable packages + """ + + if name in self.packages: + candidates = self._find_versions( + self.packages[name], version + ) + if len(candidates) > 0: + return candidates + + if name in self.obsoletes: + return self._resolve_relation( + self.obsoletes[name], version + ) + + if name in self.provides: + return self._resolve_relation( + self.provides[name], version + ) + return [] + + def add(self, package): + """Adds new package to indexes. + + :param package: the package object. + """ + self.packages[package.name][package.version] = package + key = package.name, package.version + + for obsolete in package.obsoletes: + self.obsoletes[obsolete.name][key] = obsolete + + for provide in package.provides: + self.provides[provide.name][key] = provide + + def _resolve_relation(self, relations, version): + """Resolve relation according to relations index. + + :param relations: the index of relations + :param version: the range of versions + :return: package if found, otherwise None + """ + for key, candidate in relations.iter_items(reverse=True): + if candidate.version.has_intersection(version): + return [self.packages[key[0]][key[1]]] + return [] + + @staticmethod + def _find_versions(versions, version): + """Searches accurate version. + + Search for the highest version out of intersection + of existing and required range of versions. + + :param versions: the existing versions + :param version: the required range of versions + :return: package if found, otherwise None + """ + + try: + op = Index.operators[version.op] + except KeyError: + raise ValueError( + "Unsupported operation: {0}" + .format(version.op) + ) + return op(versions, version.edge) diff --git a/packetary/objects/package.py b/packetary/objects/package.py new file mode 100644 index 0000000..cb570da --- /dev/null +++ b/packetary/objects/package.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from collections import namedtuple + +from packetary.objects.base import ComparableObject + + +FileChecksum = namedtuple("FileChecksum", ("md5", "sha1", "sha256")) + + +class Package(ComparableObject): + """Structure to describe package object.""" + + def __init__(self, repository, name, version, filename, + filesize, checksum, mandatory=False, + requires=None, provides=None, obsoletes=None): + """Initialises. + + :param name: the package`s name + :param version: the package`s version + :param filename: the package`s relative filename + :param filesize: the package`s file size + :param checksum: the package`s checksum + :param requires: the package`s requirements(optional) + :param provides: the package`s provides(optional) + :param obsoletes: the package`s obsoletes(optional) + :param mandatory: indicates that package is mandatory + """ + + self.repository = repository + self.name = name + self.version = version + self.filename = filename + self.checksum = checksum + self.filesize = filesize + self.requires = requires or [] + self.provides = provides or [] + self.obsoletes = obsoletes or [] + self.mandatory = mandatory + + def __copy__(self): + """Creates shallow copy of package.""" + return Package(**self.__dict__) + + def __str__(self): + return "{0} {1}".format(self.name, self.version) + + def __unicode__(self): + return u"{0} {1}".format(self.name, self.version) + + def __hash__(self): + return hash((self.name, self.version)) + + def cmp(self, other): + """Compares with other Package object.""" + if self.name < other.name: + return -1 + if self.name > other.name: + return 1 + if self.version < other.version: + return -1 + if self.version > other.version: + return 1 + return 0 diff --git a/packetary/objects/package_relation.py b/packetary/objects/package_relation.py new file mode 100644 index 0000000..fe7ba86 --- /dev/null +++ b/packetary/objects/package_relation.py @@ -0,0 +1,154 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import operator + + +class VersionRange(object): + """Describes the range of versions. + + Range of version is compare operation and edge. + the compare operation can be one of: + equal, greater, less, greater or equal, less or equal. + """ + def __init__(self, op=None, edge=None): + """Initialises. + + :param op: the name of operator to compare. + :param edge: the edge of versions. + """ + self.op = op + self.edge = edge + + def __hash__(self): + return hash((self.op, self.edge)) + + def __eq__(self, other): + if not isinstance(other, VersionRange): + return False + + return self.op == other.op and \ + self.edge == other.edge + + def __str__(self): + if self.edge is not None: + return "{0} {1}".format(self.op, self.edge) + return "any" + + def __unicode__(self): + if self.edge is not None: + return u"{0} {1}".format(self.op, self.edge) + return u"any" + + def has_intersection(self, other): + """Checks that 2 ranges has intersection.""" + + if not isinstance(other, VersionRange): + raise TypeError( + "Unorderable type and {0}" + .format(type(other)) + ) + + if self.op is None or other.op is None: + return True + + my_op = getattr(operator, self.op) + other_op = getattr(operator, other.op) + if self.op[0] == other.op[0]: + if self.op[0] == 'l': + if self.edge < other.edge: + return my_op(self.edge, other.edge) + return other_op(other.edge, self.edge) + elif self.op[0] == 'g': + if self.edge > other.edge: + return my_op(self.edge, other.edge) + return other_op(other.edge, self.edge) + + if self.op == 'eq': + return other_op(self.edge, other.edge) + + if other.op == 'eq': + return my_op(other.edge, self.edge) + + return ( + my_op(other.edge, self.edge) and + other_op(self.edge, other.edge) + ) + + +class PackageRelation(object): + """Describes the package`s relation. + + Relation includes the name of required package + and range of versions that satisfies requirement. + """ + + def __init__(self, name, version=None, alternative=None): + """Initialises. + + :param name: the name of required package + :param version: the version range of required package + :param alternative: the alternative relation + """ + self.name = name + self.version = VersionRange() if version is None else version + self.alternative = alternative + + @classmethod + def from_args(cls, *args): + """Construct relation from list of arguments. + + :param args: the list of tuples(name, [version_op, version_edge]) + """ + if len(args) == 0: + return None + + head = args[0] + name = head[0] + version = VersionRange(*head[1:]) + alternative = cls.from_args(*args[1:]) + return cls(name, version, alternative) + + def __iter__(self): + """Iterates over alternatives.""" + r = self + while r is not None: + yield r + r = r.alternative + + def __hash__(self): + return hash((self.name, self.version)) + + def __eq__(self, other): + if not isinstance(other, PackageRelation): + return False + + return self.name == other.name and \ + self.version == other.version + + def __str__(self): + if self.alternative is None: + return "{0} ({1})".format(self.name, self.version) + return "{0} ({1}) | {2}".format( + self.name, self.version, self.alternative + ) + + def __unicode__(self): + if self.alternative is None: + return u"{0} ({1})".format(self.name, self.version) + return u"{0} ({1}) | {2}".format( + self.name, self.version, self.alternative + ) diff --git a/packetary/objects/repository.py b/packetary/objects/repository.py new file mode 100644 index 0000000..5ba604b --- /dev/null +++ b/packetary/objects/repository.py @@ -0,0 +1,44 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +class Repository(object): + """Structure to describe repository object.""" + + def __init__(self, name, url, architecture, origin): + """Initialises. + + :param name: the repository`s name, may be tuple of strings + :param url: the repository`s URL + :param architecture: the repository`s architecture + :param origin: the repository`s origin + """ + self.name = name + self.url = url + self.architecture = architecture + self.origin = origin + + def __str__(self): + if isinstance(self.name, tuple): + return ".".join(self.name) + return str(self.name) + + def __unicode__(self): + if isinstance(self.name, tuple): + return u".".join(self.name) + return unicode(self.name, "utf8") + + def __copy__(self): + """Creates shallow copy of package.""" + return Repository(**self.__dict__) diff --git a/packetary/tests/stubs/__init__.py b/packetary/tests/stubs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/packetary/tests/stubs/generator.py b/packetary/tests/stubs/generator.py new file mode 100644 index 0000000..6253313 --- /dev/null +++ b/packetary/tests/stubs/generator.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from packetary import objects + + +def gen_repository(name="test", url="file:///test", + architecture="x86_64", origin="Test"): + """Helper to create Repository object with default attributes.""" + return objects.Repository(name, url, architecture, origin) + + +def gen_relation(name="test", version=None): + """Helper to create PackageRelation object with default attributes.""" + return [ + objects.PackageRelation( + name=name, version=objects.VersionRange(*(version or [])) + ) + ] + + +def gen_package(idx=1, **kwargs): + """Helper to create Package object with default attributes.""" + repository = gen_repository() + kwargs.setdefault("name", "package{0}".format(idx)) + kwargs.setdefault("repository", repository) + kwargs.setdefault("version", 1) + kwargs.setdefault("checksum", objects.FileChecksum("1", "2", "3")) + kwargs.setdefault("filename", "test.pkg") + kwargs.setdefault("filesize", 1) + for relation in ("requires", "provides", "obsoletes"): + if relation not in kwargs: + kwargs[relation] = gen_relation( + "{0}{1}".format(relation, idx), ["le", idx + 1] + ) + + return objects.Package(**kwargs) diff --git a/packetary/tests/test_connections.py b/packetary/tests/test_connections.py index 33d4ef2..fc19b79 100644 --- a/packetary/tests/test_connections.py +++ b/packetary/tests/test_connections.py @@ -22,127 +22,121 @@ from packetary.library import connections from packetary.tests import base -class TestConnectionsPool(base.TestCase): - def test_get_connection(self): - pool = connections.ConnectionsPool(count=2) - self.assertEqual(2, pool.free.qsize()) - with pool.get(): - self.assertEqual(1, pool.free.qsize()) - self.assertEqual(2, pool.free.qsize()) +@mock.patch("packetary.library.connections.logger") +class TestConnectionManager(base.TestCase): + def _check_proxies(self, manager, http_proxy, https_proxy): + for h in manager.opener.handlers: + if isinstance(h, connections.urllib.ProxyHandler): + self.assertEqual( + (http_proxy, https_proxy), + (h.proxies["http"], h.proxies["https"]) + ) + break + else: + self.fail("ProxyHandler should be in list of handlers.") - def _check_proxies(self, pool, http_proxy, https_proxy): - with pool.get() as c: - for h in c.opener.handlers: - if isinstance(h, connections.urllib_request.ProxyHandler): - self.assertEqual( - (http_proxy, https_proxy), - (h.proxies["http"], h.proxies["https"]) - ) - break - else: - self.fail("ProxyHandler should be in list of handlers.") - - def test_set_proxy(self): - pool = connections.ConnectionsPool(count=1, proxy="http://localhost") - self._check_proxies(pool, "http://localhost", "http://localhost") - pool = connections.ConnectionsPool( + def test_set_proxy(self, _): + manager = connections.ConnectionsManager(proxy="http://localhost") + self._check_proxies( + manager, "http://localhost", "http://localhost" + ) + manager = connections.ConnectionsManager( proxy="http://localhost", secure_proxy="https://localhost") - self._check_proxies(pool, "http://localhost", "https://localhost") + self._check_proxies( + manager, "http://localhost", "https://localhost" + ) + manager = connections.ConnectionsManager(retries_num=2) + self.assertEqual(2, manager.retries_num) + for h in manager.opener.handlers: + if isinstance(h, connections.RetryHandler): + break + else: + self.fail("RetryHandler should be in list of handlers.") - def test_reliability(self): - pool = connections.ConnectionsPool(count=0, retries_num=2) - self.assertEqual(1, pool.free.qsize()) - with pool.get() as c: - self.assertEqual(2, c.retries_num) - for h in c.opener.handlers: - if isinstance(h, connections.RetryHandler): - break - else: - self.fail("RetryHandler should be in list of handlers.") - - -class TestConnection(base.TestCase): - def setUp(self): - super(TestConnection, self).setUp() - self.connection = connections.Connection(mock.MagicMock(), 2) - - def test_make_request(self): - request = self.connection.make_request("/test/file", 0) + @mock.patch("packetary.library.connections.urllib.build_opener") + def test_make_request(self, *_): + manager = connections.ConnectionsManager(retries_num=2) + request = manager.make_request("/test/file", 0) self.assertIsInstance(request, connections.RetryableRequest) self.assertEqual("file:///test/file", request.get_full_url()) self.assertEqual(0, request.offset) self.assertEqual(2, request.retries_left) - request2 = self.connection.make_request("http://server/path", 100) + request2 = manager.make_request("http://server/path", 100) self.assertEqual("http://server/path", request2.get_full_url()) self.assertEqual(100, request2.offset) - def test_open_stream(self): - self.connection.open_stream("/test/file") - self.assertEqual(1, self.connection.opener.open.call_count) - args = self.connection.opener.open.call_args[0] + @mock.patch("packetary.library.connections.urllib.build_opener") + def test_open_stream(self, *_): + manager = connections.ConnectionsManager(retries_num=2) + manager.open_stream("/test/file") + self.assertEqual(1, manager.opener.open.call_count) + args = manager.opener.open.call_args[0] self.assertIsInstance(args[0], connections.RetryableRequest) self.assertEqual(2, args[0].retries_left) - @mock.patch("packetary.library.connections.logger") - def test_retries_on_io_error(self, logger): - self.connection.opener.open.side_effect = [ + @mock.patch("packetary.library.connections.urllib.build_opener") + def test_retries_on_io_error(self, _, logger): + manager = connections.ConnectionsManager(retries_num=2) + manager.opener.open.side_effect = [ IOError("I/O error"), mock.MagicMock() ] - self.connection.open_stream("/test/file") - self.assertEqual(2, self.connection.opener.open.call_count) + manager.open_stream("/test/file") + self.assertEqual(2, manager.opener.open.call_count) logger.exception.assert_called_with( "Failed to open url - %s: %s. retries left - %d.", "/test/file", "I/O error", 1 ) - self.connection.opener.open.side_effect = IOError("I/O error") + manager.opener.open.side_effect = IOError("I/O error") with self.assertRaises(IOError): - self.connection.open_stream("/test/file") + manager.open_stream("/test/file") logger.exception.assert_called_with( "Failed to open url - %s: %s. retries left - %d.", "/test/file", "I/O error", 0 ) - def test_raise_other_errors(self): - self.connection.opener.open.side_effect = \ - connections.urllib_error.HTTPError("", 500, "", {}, None) + @mock.patch("packetary.library.connections.urllib.build_opener") + def test_raise_other_errors(self, *_): + manager = connections.ConnectionsManager() + manager.opener.open.side_effect = \ + connections.urlerror.HTTPError("", 500, "", {}, None) - with self.assertRaises(connections.urllib_error.URLError): - self.connection.open_stream("/test/file") + with self.assertRaises(connections.urlerror.URLError): + manager.open_stream("/test/file") - self.assertEqual(1, self.connection.opener.open.call_count) + self.assertEqual(1, manager.opener.open.call_count) + @mock.patch("packetary.library.connections.urllib.build_opener") @mock.patch("packetary.library.connections.os") - def test_retrieve_from_offset(self, os): + def test_retrieve_from_offset(self, os, *_): + manager = connections.ConnectionsManager() os.path.mkdirs.side_effect = OSError(17, "") os.open.return_value = 1 response = mock.MagicMock() - self.connection.opener.open.return_value = response + manager.opener.open.return_value = response response.read.side_effect = [b"test", b""] - self.connection.retrieve("/file/src", "/file/dst", 10) + manager.retrieve("/file/src", "/file/dst", 10) os.lseek.assert_called_once_with(1, 10, os.SEEK_SET) os.ftruncate.assert_called_once_with(1, 10) self.assertEqual(1, os.write.call_count) os.fsync.assert_called_once_with(1) os.close.assert_called_once_with(1) - @mock.patch.multiple( - "packetary.library.connections", - logger=mock.DEFAULT, - os=mock.DEFAULT - ) - def test_retrieve_from_offset_fail(self, os, logger): - os.path.mkdirs.side_effect = OSError(17, "") + @mock.patch("packetary.library.connections.urllib.build_opener") + @mock.patch("packetary.library.connections.os") + def test_retrieve_from_offset_fail(self, os, _, logger): + manager = connections.ConnectionsManager(retries_num=2) + os.path.mkdirs.side_effect = OSError(connections.errno.EACCES, "") os.open.return_value = 1 response = mock.MagicMock() - self.connection.opener.open.side_effect = [ + manager.opener.open.side_effect = [ connections.RangeError("error"), response ] response.read.side_effect = [b"test", b""] - self.connection.retrieve("/file/src", "/file/dst", 10) + manager.retrieve("/file/src", "/file/dst", 10) logger.warning.assert_called_once_with( - "Failed to resume download, starts from begin: %s", + "Failed to resume download, starts from the beginning: %s", "/file/src" ) os.lseek.assert_called_once_with(1, 0, os.SEEK_SET) diff --git a/packetary/tests/test_index.py b/packetary/tests/test_index.py new file mode 100644 index 0000000..4278679 --- /dev/null +++ b/packetary/tests/test_index.py @@ -0,0 +1,189 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import six + +from packetary.objects.index import Index + +from packetary import objects +from packetary.tests import base +from packetary.tests.stubs.generator import gen_package +from packetary.tests.stubs.generator import gen_relation + + +class TestIndex(base.TestCase): + def test_add(self): + index = Index() + index.add(gen_package(version=1)) + self.assertIn("package1", index.packages) + self.assertIn(1, index.packages["package1"]) + self.assertIn("obsoletes1", index.obsoletes) + self.assertIn("provides1", index.provides) + + index.add(gen_package(version=2)) + self.assertEqual(1, len(index.packages)) + self.assertIn(1, index.packages["package1"]) + self.assertIn(2, index.packages["package1"]) + self.assertEqual(1, len(index.obsoletes)) + self.assertEqual(1, len(index.provides)) + + def test_find(self): + index = Index() + p1 = gen_package(version=1) + p2 = gen_package(version=2) + index.add(p1) + index.add(p2) + + self.assertIs( + p1, + index.find("package1", objects.VersionRange("eq", 1)) + ) + self.assertIs( + p2, + index.find("package1", objects.VersionRange()) + ) + self.assertIsNone( + index.find("package1", objects.VersionRange("gt", 2)) + ) + + def test_find_all(self): + index = Index() + p11 = gen_package(idx=1, version=1) + p12 = gen_package(idx=1, version=2) + p21 = gen_package(idx=2, version=1) + p22 = gen_package(idx=2, version=2) + index.add(p11) + index.add(p12) + index.add(p21) + index.add(p22) + + self.assertItemsEqual( + [p11, p12], + index.find_all("package1", objects.VersionRange()) + ) + self.assertItemsEqual( + [p21, p22], + index.find_all("package2", objects.VersionRange("le", 2)) + ) + + def test_find_newest_package(self): + index = Index() + p1 = gen_package(idx=1, version=2) + p2 = gen_package(idx=2, version=2) + p2.obsoletes.extend( + gen_relation(p1.name, ["lt", p1.version]) + ) + index.add(p1) + index.add(p2) + + self.assertIs( + p1, index.find(p1.name, objects.VersionRange("eq", p1.version)) + ) + self.assertIs( + p2, index.find(p1.name, objects.VersionRange("eq", 1)) + ) + + def test_find_top_down(self): + index = Index() + p1 = gen_package(version=1) + p2 = gen_package(version=2) + index.add(p1) + index.add(p2) + self.assertIs( + p2, + index.find("package1", objects.VersionRange("le", 2)) + ) + self.assertIs( + p1, + index.find("package1", objects.VersionRange("lt", 2)) + ) + self.assertIsNone( + index.find("package1", objects.VersionRange("lt", 1)) + ) + + def test_find_down_up(self): + index = Index() + p1 = gen_package(version=1) + p2 = gen_package(version=2) + index.add(p1) + index.add(p2) + self.assertIs( + p2, + index.find("package1", objects.VersionRange("ge", 2)) + ) + self.assertIs( + p2, + index.find("package1", objects.VersionRange("gt", 1)) + ) + self.assertIsNone( + index.find("package1", objects.VersionRange("gt", 2)) + ) + + def test_find_accurate(self): + index = Index() + p1 = gen_package(version=1) + p2 = gen_package(version=2) + index.add(p1) + index.add(p2) + self.assertIs( + p1, + index.find("package1", objects.VersionRange("eq", 1)) + ) + self.assertIsNone( + index.find("package1", objects.VersionRange("eq", 3)) + ) + + def test_find_obsolete(self): + index = Index() + p1 = gen_package(version=1) + index.add(p1) + + self.assertIs( + p1, index.find("obsoletes1", objects.VersionRange("le", 2)) + ) + self.assertIsNone( + index.find("obsoletes1", objects.VersionRange("gt", 2)) + ) + + def test_find_provides(self): + index = Index() + p1 = gen_package(version=1) + p2 = gen_package(version=2) + index.add(p1) + index.add(p2) + + self.assertIs( + p2, index.find("provides1", objects.VersionRange("ge", 2)) + ) + self.assertIsNone( + index.find("provides1", objects.VersionRange("lt", 2)) + ) + + def test_len(self): + index = Index() + for i in six.moves.range(3): + index.add(gen_package(idx=i + 1)) + self.assertEqual(3, len(index)) + + for i in six.moves.range(3): + index.add(gen_package(idx=i + 1, version=2)) + self.assertEqual(6, len(index)) + self.assertEqual(3, len(index.packages)) + + for i in six.moves.range(3): + index.add(gen_package(idx=i + 1, version=2)) + self.assertEqual(6, len(index)) + self.assertEqual(3, len(index.packages)) diff --git a/packetary/tests/test_objects.py b/packetary/tests/test_objects.py new file mode 100644 index 0000000..a582346 --- /dev/null +++ b/packetary/tests/test_objects.py @@ -0,0 +1,183 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import copy +import six + +from packetary.objects import PackageRelation +from packetary.objects import VersionRange + +from packetary.tests import base +from packetary.tests.stubs import generator + + +class TestObjectBase(base.TestCase): + def check_copy(self, origin): + clone = copy.copy(origin) + self.assertIsNot(origin, clone) + self.assertEqual(origin, clone) + origin_name = origin.name + origin.name += "1" + self.assertEqual( + origin_name, + clone.name + ) + + def check_ordering(self, *args): + for i in six.moves.range(len(args) - 1, 1, -1): + self.assertLess(args[i - 1], args[i]) + self.assertGreater(args[i], args[i - 1]) + + def check_equal(self, o1, o11, o2): + self.assertEqual(o1, o11) + self.assertEqual(o11, o1) + self.assertNotEqual(o1, o2) + self.assertNotEqual(o2, o1) + self.assertNotEqual(o1, None) + + def check_hashable(self, o1, o2): + d = dict() + d[o1] = o2 + d[o2] = o1 + + self.assertIs(o2, d[o1]) + self.assertIs(o1, d[o2]) + + +class TestPackageObject(TestObjectBase): + def test_copy(self): + self.check_copy(generator.gen_package(name="test1")) + + def test_ordering(self): + self.check_ordering([ + generator.gen_package(name="test1", version=1), + generator.gen_package(name="test1", version=2), + generator.gen_package(name="test2", version=1), + generator.gen_package(name="test2", version=2) + ]) + + def test_equal(self): + self.check_equal( + generator.gen_package(name="test1", version=1), + generator.gen_package(name="test1", version=1), + generator.gen_package(name="test2", version=1) + ) + + def test_hashable(self): + self.check_hashable( + generator.gen_package(name="test1", version=1), + generator.gen_package(name="test2", version=1), + ) + self.check_hashable( + generator.gen_package(name="test1", version=1), + generator.gen_package(name="test1", version=2), + ) + + +class TestRepositoryObject(base.TestCase): + def test_copy(self): + origin = generator.gen_repository() + clone = copy.copy(origin) + self.assertEqual(clone.name, origin.name) + self.assertEqual(clone.architecture, origin.architecture) + + +class TestRelationObject(TestObjectBase): + def test_equal(self): + self.check_equal( + generator.gen_relation(name="test1"), + generator.gen_relation(name="test1"), + generator.gen_relation(name="test2") + ) + + def test_hashable(self): + self.check_hashable( + generator.gen_relation(name="test1")[0], + generator.gen_relation(name="test1", version=["le", 1])[0] + ) + + def test_from_args(self): + r = PackageRelation.from_args( + ("test", "le", 2), ("test2",), ("test3",) + ) + self.assertEqual("test", r.name) + self.assertEqual("le", r.version.op) + self.assertEqual(2, r.version.edge) + self.assertEqual("test2", r.alternative.name) + self.assertEqual(VersionRange(), r.alternative.version) + self.assertEqual("test3", r.alternative.alternative.name) + self.assertEqual(VersionRange(), r.alternative.alternative.version) + self.assertIsNone(r.alternative.alternative.alternative) + + def test_iter(self): + it = iter(PackageRelation.from_args( + ("test", "le", 2), ("test2", "ge", 3)) + ) + self.assertEqual("test", next(it).name) + self.assertEqual("test2", next(it).name) + with self.assertRaises(StopIteration): + next(it) + + +class TestVersionRange(TestObjectBase): + def test_equal(self): + self.check_equal( + generator.gen_relation(name="test1"), + generator.gen_relation(name="test1"), + generator.gen_relation(name="test2") + ) + + def test_hashable(self): + self.check_hashable( + VersionRange(op="le"), + VersionRange(op="le", edge=3) + ) + + def __check_intersection(self, assertion, cases): + for data in cases: + v1 = VersionRange(*data[0]) + v2 = VersionRange(*data[1]) + assertion( + v1.has_intersection(v2), msg="%s and %s" % (v1, v2) + ) + assertion( + v2.has_intersection(v1), msg="%s and %s" % (v2, v1) + ) + + def test_have_intersection(self): + cases = [ + (("lt", 2), ("gt", 1)), + (("lt", 3), ("lt", 4)), + (("gt", 3), ("gt", 4)), + (("eq", 1), ("eq", 1)), + (("ge", 1), ("le", 1)), + (("eq", 1), ("lt", 2)), + ((None, None), ("le", 10)), + ] + self.__check_intersection(self.assertTrue, cases) + + def test_does_not_have_intersection(self): + cases = [ + (("lt", 2), ("gt", 2)), + (("ge", 2), ("lt", 2)), + (("gt", 2), ("le", 2)), + (("gt", 1), ("lt", 1)), + ] + self.__check_intersection(self.assertFalse, cases) + + def test_intersection_is_typesafe(self): + with self.assertRaises(TypeError): + VersionRange("eq", 1).has_intersection(("eq", 1)) diff --git a/requirements.txt b/requirements.txt index 2aee4db..07ceb85 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,9 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -Babel>=1.3 -eventlet>=0.17 pbr>=1.6 +Babel>=1.3 +eventlet>=0.15 +bintrees>=2.0.2 +chardet>=2.3.0 +six>=1.5.2