Add methods to get all of the version data

We've got great discovery support, but if someone is wanting to find out
what is available and doesn't otherwise know what they're looking for,
they're out of luck.

Add a method to EndpointData which will return all of the version data
for a given service, and then add a method to the base auth plugin that
will use that method to collect all of the version discovery documents
for every service in the cloud.

This commit adds os-service-types so that the resulting datastructure
can return only official service type keys. A followup patch will also
use os-service-types to allow catalog lookups by service-type alias.

There is a change to the test_identity_common.V2.get_auth_data method
to remove the public and internal urls for keystone from the catalog.
The V3 catalog only has keystone on admin, so this makes them have
equivilent data.

Change-Id: I07243edb939865a5df8b283e7c626874ffd830db
This commit is contained in:
Monty Taylor 2018-04-05 14:16:32 -05:00
parent 79cd91e755
commit d6670ee5c9
No known key found for this signature in database
GPG Key ID: 7BAE94BC7141A594
4 changed files with 498 additions and 2 deletions

View File

@ -555,6 +555,37 @@ class Discover(object):
versions.sort(key=lambda v: v['version'], reverse=reverse)
return versions
def version_string_data(self, reverse=False, **kwargs):
"""Get normalized version data with versions as strings.
Return version data in a structured way.
:param bool reverse: Reverse the list. reverse=true will mean the
returned list is sorted from newest to oldest
version.
:returns: A list of version data dictionaries sorted by version number.
Each data element in the returned list is a dictionary
consisting of:
:version string: The normalized version of the endpoint.
:url str: The url for the endpoint.
:collection: The URL for the discovery document. May be None.
:min_microversion str: The minimum microversion supported by the
endpoint. May be None.
:max_microversion str: The maximum microversion supported by the
endpoint. May be None.
:status str: A canonicalized version of the status. Valid values
are CURRENT, SUPPORTED, DEPRECATED and EXPERIMENTAL
:raw_status str: The status as provided by the server
:rtype: list(dict)
"""
version_data = self.version_data(reverse=reverse, **kwargs)
for version in version_data:
for key in ('version', 'min_microversion', 'max_microversion'):
if version[key]:
version[key] = version_to_string(version[key])
return version_data
def data_for(self, version, **kwargs):
"""Return endpoint data for a version.
@ -869,6 +900,85 @@ class EndpointData(object):
max_version=max_version)
return new_data
def get_all_version_string_data(self, session, project_id=None):
"""Return version data for all versions discovery can find.
:param string project_id: ID of the currently scoped project. Used for
removing project_id components of URLs from
the catalog. (optional)
:returns: A list of version data dictionaries sorted by version number.
Each data element in the returned list is a dictionary
consisting of:
:version string: The normalized version of the endpoint.
:url str: The url for the endpoint.
:collection: The URL for the discovery document. May be None.
:min_microversion: The minimum microversion supported by the
endpoint. May be None.
:max_microversion: The maximum microversion supported by the
endpoint. May be None.
:status str: A canonicalized version of the status. Valid values
are CURRENT, SUPPORTED, DEPRECATED and EXPERIMENTAL
:raw_status str: The status as provided by the server
:rtype: list(dict)
"""
versions = []
for vers_url in self._get_discovery_url_choices(project_id=project_id):
try:
d = get_discovery(session, vers_url)
except Exception as e:
# Ignore errors here - we're just searching for one of the
# URLs that will give us data.
_LOGGER.debug(
"Failed attempt at discovery on %s: %s", vers_url, str(e))
continue
for version in d.version_string_data():
versions.append(version)
break
return versions or self._infer_version_data(project_id)
def _infer_version_data(self, project_id=None):
"""Return version data dict for when discovery fails.
:param string project_id: ID of the currently scoped project. Used for
removing project_id components of URLs from
the catalog. (optional)
:returns: A list of version data dictionaries sorted by version number.
Each data element in the returned list is a dictionary
consisting of:
:version string: The normalized version of the endpoint.
:url str: The url for the endpoint.
:collection: The URL for the discovery document. May be None.
:min_microversion: The minimum microversion supported by the
endpoint. May be None.
:max_microversion: The maximum microversion supported by the
endpoint. May be None.
:status str: A canonicalized version of the status. Valid values
are CURRENT, SUPPORTED, DEPRECATED and EXPERIMENTAL
:raw_status str: The status as provided by the server
:rtype: list(dict)
"""
version = self.api_version
if version:
version = version_to_string(self.api_version)
url = self.url.rstrip("/")
if project_id and url.endswith(project_id):
url, _ = self.url.rsplit('/', 1)
url += "/"
return [{
'version': version,
'collection': None,
'max_microversion': None,
'min_microversion': None,
'next_min_version': None,
'not_before': None,
'status': 'CURRENT',
'raw_status': None,
'url': url,
}]
def _set_version_info(self, session, allow=None, cache=None,
allow_version_hack=True, project_id=None,
discover_versions=False,

View File

@ -507,6 +507,62 @@ class BaseIdentityPlugin(plugin.BaseAuthPlugin):
return None
return data.api_version
def get_all_version_data(self, session, interface='public',
region_name=None, **kwargs):
"""Get version data for all services in the catalog.
:param session: A session object that can be used for communication.
:type session: keystoneauth1.session.Session
:param interface:
Type of endpoint to get version data for. Can be a single value
or a list of values. A value of None indicates that all interfaces
should be queried. (optional, defaults to public)
:param string region_name:
Region of endpoints to get version data for. A valueof None
indicates that all regions should be queried. (optional, defaults
to None)
:returns: A dictionary keyed by region_name with values containing
dictionaries keyed by interface with values being a list of
version data dictionaries. Each version data dictionary consists
of:
:version string: The normalized version of the endpoint.
:url str: The url for the endpoint.
:collection: The URL for the discovery document. May be None.
:min_microversion: The minimum microversion supported by the
endpoint. May be None.
:max_microversion: The maximum microversion supported by the
endpoint. May be None.
:status str: A canonicalized version of the status. Valid values
are CURRENT, SUPPORTED, DEPRECATED and EXPERIMENTAL
:raw_status str: The status as provided by the server
"""
service_types = discover._SERVICE_TYPES
catalog = self.get_access(session).service_catalog
version_data = {}
endpoints_data = catalog.get_endpoints_data(
interface=interface, region_name=region_name)
for service_type, services in endpoints_data.items():
if service_types.is_known(service_type):
service_type = service_types.get_service_type(service_type)
for service in services:
versions = service.get_all_version_string_data(
session=session,
project_id=self.get_project_id(session),
)
if service.region_name not in version_data:
version_data[service.region_name] = {}
regions = version_data[service.region_name]
interface = service.interface.rstrip('URL')
if interface not in regions:
regions[interface] = {}
regions[interface][service_type] = versions
return version_data
def get_user_id(self, session, **kwargs):
return self.get_access(session).user_id

View File

@ -1048,6 +1048,42 @@ class Session(object):
auth = self._auth_required(auth, 'determine endpoint URL')
return auth.get_api_major_version(self, **kwargs)
def get_all_version_data(self, auth=None, interface=None,
region_name=None, **kwargs):
"""Get version data for all services in the catalog.
:param auth:
The auth plugin to use for token. Overrides the plugin on
the session. (optional)
:type auth: keystoneauth1.plugin.BaseAuthPlugin
:param interface:
Type of endpoint to get version data for. Can be a single value
or a list of values. A value of None indicates that all interfaces
should be queried. (optional, defaults to public)
:param string region_name:
Region of endpoints to get version data for. A valueof None
indicates that all regions should be queried. (optional, defaults
to None)
:returns: A dictionary keyed by region_name with values containing
dictionaries keyed by interface with values being a list of
version data dictionaries. Each version data dictionary consists
of:
:version string: The normalized version of the endpoint.
:url str: The url for the endpoint.
:collection: The URL for the discovery document. May be None.
:min_microversion: The minimum microversion supported by the
endpoint. May be None.
:max_microversion: The maximum microversion supported by the
endpoint. May be None.
:status str: A canonicalized version of the status. Valid values
are CURRENT, SUPPORTED, DEPRECATED and EXPERIMENTAL
:raw_status str: The status as provided by the server
"""
auth = self._auth_required(auth, 'determine endpoint URL')
return auth.get_all_version_data(
self, interface=interface, region_name=region_name, **kwargs)
def get_auth_connection_params(self, auth=None, **kwargs):
"""Return auth connection params as provided by the auth plugin.
@ -1148,7 +1184,6 @@ class Session(object):
auth = self._auth_required(auth, 'get project_id')
return auth.get_project_id(self)
REQUESTS_VERSION = tuple(int(v) for v in requests.__version__.split('.'))

View File

@ -579,6 +579,292 @@ class CommonIdentityTests(object):
# We should have gotten the version from the URL
self.assertEqual((3, 0), data.api_version)
def test_get_all_version_data_all_interfaces(self):
for interface in ('public', 'internal', 'admin'):
# The version discovery dict will not have a project_id
disc = fixture.DiscoveryList(v2=False, v3=False)
disc.add_nova_microversion(
href=getattr(self.TEST_VOLUME.versions['v3'].discovery,
interface),
id='v3.0', status='CURRENT',
min_version='3.0', version='3.20')
# Adding a v2 version to a service named volumev3 is not
# an error. The service itself is cinder and has more than
# one major version.
disc.add_nova_microversion(
href=getattr(self.TEST_VOLUME.versions['v2'].discovery,
interface),
id='v2.0', status='SUPPORTED')
self.stub_url(
'GET', [],
base_url=getattr(self.TEST_VOLUME.unversioned,
interface) + '/',
json=disc)
for url in (
self.TEST_COMPUTE_PUBLIC,
self.TEST_COMPUTE_INTERNAL,
self.TEST_COMPUTE_ADMIN):
disc = fixture.DiscoveryList(v2=False, v3=False)
disc.add_microversion(
href=url, id='v2')
disc.add_microversion(
href=url, id='v2.1',
min_version='2.1', max_version='2.35')
self.stub_url('GET', [], base_url=url, json=disc)
a = self.create_auth_plugin()
s = session.Session(auth=a)
identity_endpoint = 'http://127.0.0.1:35357/{}/'.format(self.version)
data = s.get_all_version_data(interface=None)
self.assertEqual({
'RegionOne': {
'admin': {
'block-storage': [{
'collection': None,
'max_microversion': None,
'min_microversion': None,
'next_min_version': None,
'not_before': None,
'raw_status': 'SUPPORTED',
'status': 'SUPPORTED',
'url': 'https://block-storage.example.com/admin/v2',
'version': '2.0'
}, {
'collection': None,
'max_microversion': '3.20',
'min_microversion': '3.0',
'next_min_version': None,
'not_before': None,
'raw_status': 'CURRENT',
'status': 'CURRENT',
'url': 'https://block-storage.example.com/admin/v3',
'version': '3.0'
}],
'compute': [{
'collection': None,
'max_microversion': None,
'min_microversion': None,
'next_min_version': None,
'not_before': None,
'raw_status': 'stable',
'status': 'CURRENT',
'url': 'https://compute.example.com/nova/admin',
'version': '2.0'
}, {
'collection': None,
'max_microversion': '2.35',
'min_microversion': '2.1',
'next_min_version': None,
'not_before': None,
'raw_status': 'stable',
'status': 'CURRENT',
'url': 'https://compute.example.com/nova/admin',
'version': '2.1'}],
'identity': [{
'collection': None,
'max_microversion': None,
'min_microversion': None,
'next_min_version': None,
'not_before': None,
'raw_status': None,
'status': 'CURRENT',
'url': identity_endpoint,
'version': self.discovery_version,
}]
},
'internal': {
'baremetal': [{
'collection': None,
'max_microversion': None,
'min_microversion': None,
'next_min_version': None,
'not_before': None,
'raw_status': None,
'status': 'CURRENT',
'url': 'https://baremetal.example.com/internal/',
'version': None
}],
'block-storage': [{
'collection': None,
'max_microversion': None,
'min_microversion': None,
'next_min_version': None,
'not_before': None,
'raw_status': 'SUPPORTED',
'status': 'SUPPORTED',
'url': 'https://block-storage.example.com/internal/v2',
'version': '2.0'
}, {
'collection': None,
'max_microversion': '3.20',
'min_microversion': '3.0',
'next_min_version': None,
'not_before': None,
'raw_status': 'CURRENT',
'status': 'CURRENT',
'url': 'https://block-storage.example.com/internal/v3',
'version': '3.0'
}],
'compute': [{
'collection': None,
'max_microversion': None,
'min_microversion': None,
'next_min_version': None,
'not_before': None,
'raw_status': 'stable',
'status': 'CURRENT',
'url': 'https://compute.example.com/nova/internal',
'version': '2.0'
}, {
'collection': None,
'max_microversion': '2.35',
'min_microversion': '2.1',
'next_min_version': None,
'not_before': None,
'raw_status': 'stable',
'status': 'CURRENT',
'url': 'https://compute.example.com/nova/internal',
'version': '2.1'
}]
},
'public': {
'block-storage': [{
'collection': None,
'max_microversion': None,
'min_microversion': None,
'next_min_version': None,
'not_before': None,
'raw_status': 'SUPPORTED',
'status': 'SUPPORTED',
'url': 'https://block-storage.example.com/public/v2',
'version': '2.0'
}, {
'collection': None,
'max_microversion': '3.20',
'min_microversion': '3.0',
'next_min_version': None,
'not_before': None,
'raw_status': 'CURRENT',
'status': 'CURRENT',
'url': 'https://block-storage.example.com/public/v3',
'version': '3.0'
}],
'compute': [{
'collection': None,
'max_microversion': None,
'min_microversion': None,
'next_min_version': None,
'not_before': None,
'raw_status': 'stable',
'status': 'CURRENT',
'url': 'https://compute.example.com/nova/public',
'version': '2.0'
}, {
'collection': None,
'max_microversion': '2.35',
'min_microversion': '2.1',
'next_min_version': None,
'not_before': None,
'raw_status': 'stable',
'status': 'CURRENT',
'url': 'https://compute.example.com/nova/public',
'version': '2.1',
}]
}
}
}, data)
def test_get_all_version_data(self):
cinder_disc = fixture.DiscoveryList(v2=False, v3=False)
# The version discovery dict will not have a project_id
cinder_disc.add_nova_microversion(
href=self.TEST_VOLUME.versions['v3'].discovery.public,
id='v3.0', status='CURRENT',
min_version='3.0', version='3.20')
# Adding a v2 version to a service named volumev3 is not
# an error. The service itself is cinder and has more than
# one major version.
cinder_disc.add_nova_microversion(
href=self.TEST_VOLUME.versions['v2'].discovery.public,
id='v2.0', status='SUPPORTED')
self.stub_url(
'GET', [],
base_url=self.TEST_VOLUME.unversioned.public + '/',
json=cinder_disc)
nova_disc = fixture.DiscoveryList(v2=False, v3=False)
nova_disc.add_microversion(
href=self.TEST_COMPUTE_PUBLIC, id='v2')
nova_disc.add_microversion(
href=self.TEST_COMPUTE_PUBLIC, id='v2.1',
min_version='2.1', max_version='2.35')
self.stub_url(
'GET', [], base_url=self.TEST_COMPUTE_PUBLIC, json=nova_disc)
a = self.create_auth_plugin()
s = session.Session(auth=a)
data = s.get_all_version_data(interface='public')
self.assertEqual({
'RegionOne': {
'public': {
'block-storage': [{
'collection': None,
'max_microversion': None,
'min_microversion': None,
'next_min_version': None,
'not_before': None,
'raw_status': 'SUPPORTED',
'status': 'SUPPORTED',
'url': 'https://block-storage.example.com/public/v2',
'version': '2.0'
}, {
'collection': None,
'max_microversion': '3.20',
'min_microversion': '3.0',
'next_min_version': None,
'not_before': None,
'raw_status': 'CURRENT',
'status': 'CURRENT',
'url': 'https://block-storage.example.com/public/v3',
'version': '3.0'
}],
'compute': [{
'collection': None,
'max_microversion': None,
'min_microversion': None,
'next_min_version': None,
'not_before': None,
'raw_status': 'stable',
'status': 'CURRENT',
'url': 'https://compute.example.com/nova/public',
'version': '2.0'
}, {
'collection': None,
'max_microversion': '2.35',
'min_microversion': '2.1',
'next_min_version': None,
'not_before': None,
'raw_status': 'stable',
'status': 'CURRENT',
'url': 'https://compute.example.com/nova/public',
'version': '2.1'
}],
}
}
}, data)
def test_endpoint_data_no_version_no_discovery(self):
a = self.create_auth_plugin()
s = session.Session(auth=a)
@ -1095,6 +1381,10 @@ class V3(CommonIdentityTests, utils.TestCase):
def version(self):
return 'v3'
@property
def discovery_version(self):
return '3.0'
def get_auth_data(self, **kwargs):
kwargs.setdefault('project_id', self.PROJECT_ID)
token = fixture.V3Token(**kwargs)
@ -1150,6 +1440,10 @@ class V2(CommonIdentityTests, utils.TestCase):
def version(self):
return 'v2.0'
@property
def discovery_version(self):
return '2.0'
def create_auth_plugin(self, **kwargs):
kwargs.setdefault('auth_url', self.TEST_URL)
kwargs.setdefault('username', self.TEST_USER)
@ -1173,7 +1467,8 @@ class V2(CommonIdentityTests, utils.TestCase):
region = 'RegionOne'
svc = token.add_service('identity')
svc.add_endpoint(self.TEST_ADMIN_URL, region=region)
svc.add_endpoint(admin=self.TEST_ADMIN_URL, region=region,
public=None, internal=None)
svc = token.add_service('compute')
svc.add_endpoint(public=self.TEST_COMPUTE_PUBLIC,