diff --git a/doc/source/conf.py b/doc/source/conf.py index 792110237..ec4dab832 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -60,7 +60,13 @@ add_module_names = True # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'native' -autodoc_member_order = "bysource" +autodoc_member_order = 'bysource' + +# Include both the class and __init__ docstrings when describing the class +autoclass_content = 'both' + +# Don't document type hints as they're too noisy +autodoc_typehints = 'none' # Locations to exclude when looking for source files. exclude_patterns = [] @@ -70,8 +76,7 @@ exclude_patterns = [] # Don't let openstackdocstheme insert TOCs automatically. theme_include_auto_toc = False -# Output file base name for HTML help builder. -htmlhelp_basename = 'openstacksdkdoc' +# -- Options for LaTeX output --------------------------------------------- # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass @@ -91,6 +96,3 @@ latex_elements = {'maxlistdepth': 10} # Disable usage of xindy https://bugzilla.redhat.com/show_bug.cgi?id=1643664 latex_use_xindy = False - -# Include both the class and __init__ docstrings when describing the class -autoclass_content = "both" diff --git a/openstack/common/metadata.py b/openstack/common/metadata.py index 81a6a98ef..36c30b1d0 100644 --- a/openstack/common/metadata.py +++ b/openstack/common/metadata.py @@ -9,12 +9,17 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. + from openstack import exceptions from openstack import resource from openstack import utils class MetadataMixin: + id: resource.Body + base_path: str + _body: resource._ComponentManager + #: *Type: list of tag strings* metadata = resource.Body('metadata', type=dict) diff --git a/openstack/common/quota_set.py b/openstack/common/quota_set.py index c298416e6..f5afd7c04 100644 --- a/openstack/common/quota_set.py +++ b/openstack/common/quota_set.py @@ -9,6 +9,9 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. + +import typing as ty + from openstack import exceptions from openstack import resource @@ -88,7 +91,7 @@ class QuotaSet(resource.Resource): body.pop("self", None) # Process body_attrs to strip usage and reservation out - normalized_attrs = dict( + normalized_attrs: ty.Dict[str, ty.Any] = dict( reservation={}, usage={}, ) diff --git a/openstack/common/tag.py b/openstack/common/tag.py index 0d25693ff..e49a38268 100644 --- a/openstack/common/tag.py +++ b/openstack/common/tag.py @@ -9,12 +9,21 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. + from openstack import exceptions from openstack import resource from openstack import utils class TagMixin: + id: resource.Body + base_path: str + _body: resource._ComponentManager + + @classmethod + def _get_session(cls, session): + ... + _tag_query_parameters = { 'tags': 'tags', 'any_tags': 'tags-any', diff --git a/openstack/config/cloud_region.py b/openstack/config/cloud_region.py index ddd91bc8e..ee16dd612 100644 --- a/openstack/config/cloud_region.py +++ b/openstack/config/cloud_region.py @@ -14,6 +14,7 @@ import copy import os.path +import typing as ty import urllib import warnings @@ -195,7 +196,7 @@ def from_conf(conf, session=None, service_types=None, **kwargs): ), ) continue - opt_dict = {} + opt_dict: ty.Dict[str, str] = {} # Populate opt_dict with (appropriately processed) Adapter conf opts try: ks_load_adap.process_conf_options(conf[project_name], opt_dict) diff --git a/openstack/config/loader.py b/openstack/config/loader.py index 8df3422be..1f330e8f4 100644 --- a/openstack/config/loader.py +++ b/openstack/config/loader.py @@ -21,6 +21,7 @@ import json import os import re import sys +import typing as ty import warnings import appdirs @@ -129,7 +130,7 @@ def _fix_argv(argv): argv[index] = "=".join(split_args) # Save both for later so we can throw an error about dupes processed[new].add(orig) - overlap = [] + overlap: ty.List[str] = [] for new, old in processed.items(): if len(old) > 1: overlap.extend(old) @@ -297,8 +298,8 @@ class OpenStackConfig: self._cache_expiration_time = 0 self._cache_path = CACHE_PATH self._cache_class = 'dogpile.cache.null' - self._cache_arguments = {} - self._cache_expirations = {} + self._cache_arguments: ty.Dict[str, ty.Any] = {} + self._cache_expirations: ty.Dict[str, int] = {} self._influxdb_config = {} if 'cache' in self.cloud_config: cache_settings = _util.normalize_keys(self.cloud_config['cache']) @@ -514,8 +515,8 @@ class OpenStackConfig: return self._expand_regions(regions) else: # crappit. we don't have a region defined. - new_cloud = dict() - our_cloud = self.cloud_config['clouds'].get(cloud, dict()) + new_cloud: ty.Dict[str, ty.Any] = {} + our_cloud = self.cloud_config['clouds'].get(cloud, {}) self._expand_vendor_profile(cloud, new_cloud, our_cloud) if 'regions' in new_cloud and new_cloud['regions']: return self._expand_regions(new_cloud['regions']) diff --git a/openstack/config/vendors/__init__.py b/openstack/config/vendors/__init__.py index 95c5ccb30..189d3744a 100644 --- a/openstack/config/vendors/__init__.py +++ b/openstack/config/vendors/__init__.py @@ -15,6 +15,7 @@ import glob import json import os +import typing as ty import urllib import requests @@ -24,7 +25,7 @@ from openstack.config import _util from openstack import exceptions _VENDORS_PATH = os.path.dirname(os.path.realpath(__file__)) -_VENDOR_DEFAULTS = {} +_VENDOR_DEFAULTS: ty.Dict[str, ty.Dict] = {} _WELL_KNOWN_PATH = "{scheme}://{netloc}/.well-known/openstack/api" diff --git a/openstack/connection.py b/openstack/connection.py index 430666c16..b0f317395 100644 --- a/openstack/connection.py +++ b/openstack/connection.py @@ -209,7 +209,7 @@ try: import importlib.metadata as importlib_metadata except ImportError: # For everyone else - import importlib_metadata + import importlib_metadata # type: ignore import keystoneauth1.exceptions import requestsexceptions diff --git a/openstack/exceptions.py b/openstack/exceptions.py index 93a4b9d06..439e85123 100644 --- a/openstack/exceptions.py +++ b/openstack/exceptions.py @@ -18,6 +18,7 @@ Exception definitions. import json import re +import typing as ty from requests import exceptions as _rex @@ -214,6 +215,7 @@ def raise_from_response(response, error_message=None): if response.status_code < 400: return + cls: ty.Type[SDKException] if response.status_code == 400: cls = BadRequestException elif response.status_code == 403: @@ -251,6 +253,7 @@ def raise_from_response(response, error_message=None): message = re.sub(r'<.+?>', '', line.strip()) if message not in messages: messages.append(message) + # Return joined string separated by colons. details = ': '.join(messages) diff --git a/openstack/network/v2/_proxy.py b/openstack/network/v2/_proxy.py index a14fd4366..7aafb2649 100644 --- a/openstack/network/v2/_proxy.py +++ b/openstack/network/v2/_proxy.py @@ -9,10 +9,8 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -from typing import Generic -from typing import Optional -from typing import Type -from typing import TypeVar + +import typing as ty from openstack import exceptions from openstack.network.v2 import address_group as _address_group @@ -98,11 +96,10 @@ from openstack.network.v2 import ( ) from openstack.network.v2 import vpn_service as _vpn_service from openstack import proxy - -T = TypeVar('T') +from openstack import resource -class Proxy(proxy.Proxy, Generic[T]): +class Proxy(proxy.Proxy): _resource_registry = { "address_group": _address_group.AddressGroup, "address_scope": _address_scope.AddressScope, @@ -189,24 +186,24 @@ class Proxy(proxy.Proxy, Generic[T]): @proxy._check_resource(strict=False) def _update( self, - resource_type: Type[T], + resource_type: ty.Type[resource.Resource], value, base_path=None, if_revision=None, **attrs, - ) -> T: + ) -> resource.Resource: res = self._get_resource(resource_type, value, **attrs) return res.commit(self, base_path=base_path, if_revision=if_revision) @proxy._check_resource(strict=False) def _delete( self, - resource_type: Type[T], + resource_type: ty.Type[resource.Resource], value, ignore_missing=True, if_revision=None, **attrs, - ) -> Optional[T]: + ) -> ty.Optional[resource.Resource]: res = self._get_resource(resource_type, value, **attrs) try: diff --git a/openstack/network/v2/port.py b/openstack/network/v2/port.py index c0fba5d2d..33ea63e69 100644 --- a/openstack/network/v2/port.py +++ b/openstack/network/v2/port.py @@ -9,7 +9,8 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -from typing import List + +import typing as ty from openstack.common import tag from openstack.network.v2 import _base @@ -58,7 +59,7 @@ class Port(_base.NetworkResource, tag.TagMixin): # Properties #: Allowed address pairs list. Dictionary key ``ip_address`` is required #: and key ``mac_address`` is optional. - allowed_address_pairs: List[dict] = resource.Body( + allowed_address_pairs: ty.List[dict] = resource.Body( 'allowed_address_pairs', type=list ) #: The ID of the host where the port is allocated. In some cases, diff --git a/openstack/proxy.py b/openstack/proxy.py index ff2dd75dc..e96e2f6d8 100644 --- a/openstack/proxy.py +++ b/openstack/proxy.py @@ -11,11 +11,7 @@ # under the License. import functools -from typing import Generator -from typing import Generic -from typing import Optional -from typing import Type -from typing import TypeVar +import typing as ty import urllib from urllib.parse import urlparse @@ -24,7 +20,7 @@ try: JSONDecodeError = simplejson.scanner.JSONDecodeError except ImportError: - JSONDecodeError = ValueError + JSONDecodeError = ValueError # type: ignore import iso8601 import jmespath from keystoneauth1 import adapter @@ -33,7 +29,8 @@ from openstack import _log from openstack import exceptions from openstack import resource -T = TypeVar('T') + +ResourceType = ty.TypeVar('ResourceType', bound=resource.Resource) # The _check_resource decorator is used on Proxy methods to ensure that @@ -74,7 +71,7 @@ def normalize_metric_name(name): return name -class Proxy(adapter.Adapter, Generic[T]): +class Proxy(adapter.Adapter): """Represents a service.""" retriable_status_codes = None @@ -84,7 +81,7 @@ class Proxy(adapter.Adapter, Generic[T]): ``_status_code_retries``. """ - _resource_registry = dict() + _resource_registry: ty.Dict[str, ty.Type[resource.Resource]] = {} """Registry of the supported resourses. Dictionary of resource names (key) types (value). @@ -431,7 +428,9 @@ class Proxy(adapter.Adapter, Generic[T]): self, '_connection', getattr(self.session, '_sdk_connection', None) ) - def _get_resource(self, resource_type: Type[T], value, **attrs) -> T: + def _get_resource( + self, resource_type: ty.Type[ResourceType], value, **attrs + ) -> ResourceType: """Get a resource object to work on :param resource_type: The type of resource to operate on. This should @@ -478,8 +477,12 @@ class Proxy(adapter.Adapter, Generic[T]): return value def _find( - self, resource_type: Type[T], name_or_id, ignore_missing=True, **attrs - ) -> Optional[T]: + self, + resource_type: ty.Type[ResourceType], + name_or_id, + ignore_missing=True, + **attrs, + ) -> ty.Optional[ResourceType]: """Find a resource :param name_or_id: The name or ID of a resource to find. @@ -500,8 +503,12 @@ class Proxy(adapter.Adapter, Generic[T]): @_check_resource(strict=False) def _delete( - self, resource_type: Type[T], value, ignore_missing=True, **attrs - ): + self, + resource_type: ty.Type[ResourceType], + value, + ignore_missing=True, + **attrs, + ) -> ty.Optional[ResourceType]: """Delete a resource :param resource_type: The type of resource to delete. This should @@ -538,8 +545,12 @@ class Proxy(adapter.Adapter, Generic[T]): @_check_resource(strict=False) def _update( - self, resource_type: Type[T], value, base_path=None, **attrs - ) -> T: + self, + resource_type: ty.Type[ResourceType], + value, + base_path=None, + **attrs, + ) -> ResourceType: """Update a resource :param resource_type: The type of resource to update. @@ -563,7 +574,12 @@ class Proxy(adapter.Adapter, Generic[T]): res = self._get_resource(resource_type, value, **attrs) return res.commit(self, base_path=base_path) - def _create(self, resource_type: Type[T], base_path=None, **attrs): + def _create( + self, + resource_type: ty.Type[ResourceType], + base_path=None, + **attrs, + ) -> ResourceType: """Create a resource from attributes :param resource_type: The type of resource to create. @@ -588,8 +604,11 @@ class Proxy(adapter.Adapter, Generic[T]): return res.create(self, base_path=base_path) def _bulk_create( - self, resource_type: Type[T], data, base_path=None - ) -> Generator[T, None, None]: + self, + resource_type: ty.Type[ResourceType], + data, + base_path=None, + ) -> ty.Generator[ResourceType, None, None]: """Create a resource from attributes :param resource_type: The type of resource to create. @@ -612,13 +631,13 @@ class Proxy(adapter.Adapter, Generic[T]): @_check_resource(strict=False) def _get( self, - resource_type: Type[T], + resource_type: ty.Type[ResourceType], value=None, requires_id=True, base_path=None, skip_cache=False, **attrs, - ): + ) -> ResourceType: """Fetch a resource :param resource_type: The type of resource to get. @@ -655,12 +674,12 @@ class Proxy(adapter.Adapter, Generic[T]): def _list( self, - resource_type: Type[T], + resource_type: ty.Type[ResourceType], paginated=True, base_path=None, jmespath_filters=None, **attrs, - ) -> Generator[T, None, None]: + ) -> ty.Generator[ResourceType, None, None]: """List a resource :param resource_type: The type of resource to list. This should @@ -696,8 +715,12 @@ class Proxy(adapter.Adapter, Generic[T]): return data def _head( - self, resource_type: Type[T], value=None, base_path=None, **attrs - ): + self, + resource_type: ty.Type[ResourceType], + value=None, + base_path=None, + **attrs, + ) -> ResourceType: """Retrieve a resource's header :param resource_type: The type of resource to retrieve. diff --git a/openstack/resource.py b/openstack/resource.py index 35e9c908f..50387a579 100644 --- a/openstack/resource.py +++ b/openstack/resource.py @@ -32,10 +32,12 @@ converted into this Resource class' appropriate components and types and then returned to the caller. """ +import abc import collections import inspect import itertools import operator +import typing as ty import urllib.parse import warnings @@ -93,11 +95,11 @@ def _convert_type(value, data_type, list_type=None): return data_type() -class _BaseComponent: +class _BaseComponent(abc.ABC): # The name this component is being tracked as in the Resource - key = None + key: str # The class to be used for mappings - _map_cls = dict + _map_cls: ty.Type[ty.Mapping] = dict #: Marks the property as deprecated. deprecated = False @@ -270,6 +272,8 @@ class Computed(_BaseComponent): class _ComponentManager(collections.abc.MutableMapping): """Storage of a component type""" + attributes: ty.Dict[str, ty.Any] + def __init__(self, attributes=None, synchronized=False): self.attributes = dict() if attributes is None else attributes.copy() self._dirty = set() if synchronized else set(self.attributes.keys()) @@ -452,14 +456,15 @@ class Resource(dict): # will work properly. #: Singular form of key for resource. - resource_key = None + resource_key: ty.Optional[str] = None #: Plural form of key for resource. - resources_key = None + resources_key: ty.Optional[str] = None #: Key used for pagination links pagination_key = None #: The ID of this resource. id = Body("id") + #: The name of this resource. name = Body("name") #: The OpenStack location of this resource. @@ -469,7 +474,7 @@ class Resource(dict): _query_mapping = QueryParameters() #: The base part of the URI for this resource. - base_path = "" + base_path: str = "" #: Allow create operation for this resource. allow_create = False @@ -508,22 +513,22 @@ class Resource(dict): create_returns_body = None #: Maximum microversion to use for getting/creating/updating the Resource - _max_microversion = None + _max_microversion: ty.Optional[str] = None #: API microversion (string or None) this Resource was loaded with microversion = None _connection = None - _body = None - _header = None - _uri = None - _computed = None - _original_body = None + _body: _ComponentManager + _header: _ComponentManager + _uri: _ComponentManager + _computed: _ComponentManager + _original_body: ty.Dict[str, ty.Any] = {} _store_unknown_attrs_as_properties = False _allow_unknown_attrs_in_body = False - _unknown_attrs_in_body = None + _unknown_attrs_in_body: ty.Dict[str, ty.Any] = {} # Placeholder for aliases as dict of {__alias__:__original} - _attr_aliases = {} + _attr_aliases: ty.Dict[str, str] = {} def __init__(self, _synchronized=False, connection=None, **attrs): """The base resource @@ -1072,12 +1077,13 @@ class Resource(dict): :return: A dictionary of key/value pairs where keys are named as they exist as attributes of this class. """ + mapping: ty.Union[utils.Munch, ty.Dict] if _to_munch: mapping = utils.Munch() else: mapping = {} - components = [] + components: ty.List[ty.Type[_BaseComponent]] = [] if body: components.append(Body) if headers: @@ -1089,9 +1095,6 @@ class Resource(dict): "At least one of `body`, `headers` or `computed` must be True" ) - # isinstance stricly requires this to be a tuple - components = tuple(components) - if body and self._allow_unknown_attrs_in_body: for key in self._unknown_attrs_in_body: converted = self._attr_to_dict( @@ -1105,7 +1108,8 @@ class Resource(dict): # but is slightly different in that we're looking at an instance # and we're mapping names on this class to their actual stored # values. - for attr, component in self._attributes_iterator(components): + # NOTE: isinstance stricly requires components to be a tuple + for attr, component in self._attributes_iterator(tuple(components)): if original_names: key = component.name else: @@ -1167,6 +1171,7 @@ class Resource(dict): *, resource_request_key=None, ): + body: ty.Union[ty.Dict[str, ty.Any], ty.List[ty.Any]] if patch: if not self._store_unknown_attrs_as_properties: # Default case @@ -1592,7 +1597,7 @@ class Resource(dict): "Invalid create method: %s" % cls.create_method ) - body = [] + _body: ty.List[ty.Any] = [] resources = [] for attrs in data: # NOTE(gryf): we need to create resource objects, since @@ -1605,9 +1610,12 @@ class Resource(dict): request = resource._prepare_request( requires_id=requires_id, base_path=base_path ) - body.append(request.body) + _body.append(request.body) + + body: ty.Union[ty.Dict[str, ty.Any], ty.List[ty.Any]] = _body if prepend_key: + assert cls.resources_key body = {cls.resources_key: body} response = method( diff --git a/openstack/service_description.py b/openstack/service_description.py index dab5d0598..eab27a88f 100644 --- a/openstack/service_description.py +++ b/openstack/service_description.py @@ -11,6 +11,7 @@ # License for the specific language governing permissions and limitations # under the License. +import typing as ty import warnings import os_service_types @@ -44,11 +45,11 @@ class _ServiceDisabledProxyShim: class ServiceDescription: #: Dictionary of supported versions and proxy classes for that version - supported_versions = None + supported_versions: ty.Dict[str, ty.Type[proxy_mod.Proxy]] = {} #: main service_type to use to find this service in the catalog - service_type = None + service_type: str #: list of aliases this service might be registered as - aliases = [] + aliases: ty.List[str] = [] def __init__(self, service_type, supported_versions=None, aliases=None): """Class describing how to interact with a REST service. diff --git a/openstack/test/fakes.py b/openstack/test/fakes.py index a405e4642..857b56a56 100644 --- a/openstack/test/fakes.py +++ b/openstack/test/fakes.py @@ -67,7 +67,7 @@ def generate_fake_resource( :raises NotImplementedError: If a resource attribute specifies a ``type`` or ``list_type`` that cannot be automatically generated """ - base_attrs = dict() + base_attrs: Dict[str, Any] = {} for name, value in inspect.getmembers( resource_type, predicate=lambda x: isinstance(x, (resource.Body, resource.URI)), @@ -182,7 +182,7 @@ def generate_fake_resources( # (better) type annotations def generate_fake_proxy( service: Type[service_description.ServiceDescription], - api_version: Optional[int] = None, + api_version: Optional[str] = None, ) -> proxy.Proxy: """Generate a fake proxy for the given service type @@ -246,10 +246,10 @@ def generate_fake_proxy( ) else: api_version = list(supported_versions)[0] - elif str(api_version) not in supported_versions: + elif api_version not in supported_versions: raise ValueError( f"API version {api_version} is not supported by openstacksdk. " f"Supported API versions are: {', '.join(supported_versions)}" ) - return mock.create_autospec(supported_versions[str(api_version)]) + return mock.create_autospec(supported_versions[api_version]) diff --git a/openstack/tests/base.py b/openstack/tests/base.py index 142d80b50..88cdebcbb 100644 --- a/openstack/tests/base.py +++ b/openstack/tests/base.py @@ -13,11 +13,12 @@ # License for the specific language governing permissions and limitations # under the License. -from io import StringIO +import io import logging import os import pprint import sys +import typing as ty import fixtures from oslotest import base @@ -59,8 +60,10 @@ class TestCase(base.BaseTestCase): self.warnings = self.useFixture(os_fixtures.WarningsFixture()) + self._log_stream: ty.TextIO + if os.environ.get('OS_LOG_CAPTURE') in _TRUE_VALUES: - self._log_stream = StringIO() + self._log_stream = io.StringIO() if os.environ.get('OS_ALWAYS_LOG') in _TRUE_VALUES: self.addCleanup(self.printLogs) else: diff --git a/openstack/utils.py b/openstack/utils.py index 012fdbab8..24c409c58 100644 --- a/openstack/utils.py +++ b/openstack/utils.py @@ -16,6 +16,7 @@ import queue import string import threading import time +import typing as ty import keystoneauth1 from keystoneauth1 import adapter as ks_adapter @@ -417,7 +418,7 @@ class TinyDAG: def _start_traverse(self): """Initialize graph traversing""" self._run_in_degree = self._get_in_degree() - self._queue = queue.Queue() + self._queue: queue.Queue[str] = queue.Queue() self._done = set() self._it_cnt = len(self._graph) @@ -427,8 +428,7 @@ class TinyDAG: def _get_in_degree(self): """Calculate the in_degree (count incoming) for nodes""" - _in_degree = dict() - _in_degree = {u: 0 for u in self._graph.keys()} + _in_degree: ty.Dict[str, int] = {u: 0 for u in self._graph.keys()} for u in self._graph: for v in self._graph[u]: _in_degree[v] += 1 @@ -568,7 +568,7 @@ class Munch(dict): def munchify(x, factory=Munch): """Recursively transforms a dictionary into a Munch via copy.""" # Munchify x, using `seen` to track object cycles - seen = dict() + seen: ty.Dict[int, ty.Any] = dict() def munchify_cycles(obj): try: @@ -608,7 +608,7 @@ def unmunchify(x): """Recursively converts a Munch into a dictionary.""" # Munchify x, using `seen` to track object cycles - seen = dict() + seen: ty.Dict[int, ty.Any] = dict() def unmunchify_cycles(obj): try: