From fa80a51d0f64efc14b33409552d4231ddb244d30 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 18 Oct 2016 06:48:19 -0500 Subject: [PATCH] Add strict mode for trimming out non-API data shade defaults to returning everything under the sun in every form possible in order to ensure maximum backwards compatability - even with systems that are not shade itself. However, passthrough fields from somewhere else could change at any time. This patch adds an opt-in flag that skips returning passthrough fields anywhere other than the properties dict. Change-Id: I7071a406965ed373e77f9592eb76975400cb426b --- doc/source/model.rst | 26 +- .../notes/strict-mode-d493abc0c3e87945.yaml | 6 + shade/__init__.py | 12 +- shade/_normalize.py | 106 +++-- shade/openstackcloud.py | 7 +- shade/tests/unit/base.py | 4 + shade/tests/unit/test__utils.py | 126 ------ shade/tests/unit/test_normalize.py | 394 ++++++++++++++++++ 8 files changed, 496 insertions(+), 185 deletions(-) create mode 100644 releasenotes/notes/strict-mode-d493abc0c3e87945.yaml create mode 100644 shade/tests/unit/test_normalize.py diff --git a/doc/source/model.rst b/doc/source/model.rst index 3b50641df..0154bb9f1 100644 --- a/doc/source/model.rst +++ b/doc/source/model.rst @@ -22,6 +22,16 @@ into an attribute called 'properties'. The contents of properties are defined to be an arbitrary collection of key value pairs with no promises as to any particular key ever existing. +If a user passes `strict=True` to the shade constructor, shade will not pass +through arbitrary objects to the root of the resource, and will instead only +put them in the properties dict. If a user is worried about accidentally +writing code that depends on an attribute that is not part of the API contract, +this can be a useful tool. Keep in mind all data can still be accessed via +the properties dict, but any code touching anything in the properties dict +should be aware that the keys found there are highly user/cloud specific. +Any key that is transformed as part of the shade data model contract will +not wind up with an entry in properties - only keys that are unknown. + Location -------- @@ -154,21 +164,20 @@ A Server from Nova name=str(), image=dict() or str(), flavor=dict(), - volumes=list(), + volumes=list(), # Volume interface_ip=str(), has_config_drive=bool(), accessIPv4=str(), accessIPv6=str(), - addresses=dict(), + addresses=dict(), # string, list(Address) created=str(), key_name=str(), - metadata=dict(), - networks=dict(), + metadata=dict(), # string, string private_v4=str(), progress=int(), public_v4=str(), public_v6=str(), - security_groups=list(), + security_groups=list(), # SecurityGroup status=str(), updated=str(), user_id=str(), @@ -195,9 +204,8 @@ A Floating IP from Neutron or Nova attached=bool(), fixed_ip_address=str() or None, floating_ip_address=str() or None, - floating_network_id=str() or None, - network=str(), - port_id=str() or None, - router_id=str(), + network=str() or None, + port=str() or None, + router=str(), status=str(), properties=dict()) diff --git a/releasenotes/notes/strict-mode-d493abc0c3e87945.yaml b/releasenotes/notes/strict-mode-d493abc0c3e87945.yaml new file mode 100644 index 000000000..ea81b138b --- /dev/null +++ b/releasenotes/notes/strict-mode-d493abc0c3e87945.yaml @@ -0,0 +1,6 @@ +--- +features: + - Added 'strict' mode, which is set by passing strict=True + to the OpenStackCloud constructor. strict mode tells shade + to only return values in resources that are part of shade's + declared data model contract. diff --git a/shade/__init__.py b/shade/__init__.py index e4d6d09d7..fc7fcc22e 100644 --- a/shade/__init__.py +++ b/shade/__init__.py @@ -55,7 +55,7 @@ def simple_logging(debug=False, http_debug=False): log = _log.setup_logging('keystoneauth.identity.generic.base') -def openstack_clouds(config=None, debug=False, cloud=None): +def openstack_clouds(config=None, debug=False, cloud=None, strict=False): if not config: config = os_client_config.OpenStackConfig() try: @@ -64,6 +64,7 @@ def openstack_clouds(config=None, debug=False, cloud=None): OpenStackCloud( cloud=f.name, debug=debug, cloud_config=f, + strict=strict, **f.config) for f in config.get_all_clouds() ] @@ -72,6 +73,7 @@ def openstack_clouds(config=None, debug=False, cloud=None): OpenStackCloud( cloud=f.name, debug=debug, cloud_config=f, + strict=strict, **f.config) for f in config.get_all_clouds() if f.name == cloud @@ -81,7 +83,7 @@ def openstack_clouds(config=None, debug=False, cloud=None): "Invalid cloud configuration: {exc}".format(exc=str(e))) -def openstack_cloud(config=None, **kwargs): +def openstack_cloud(config=None, strict=False, **kwargs): if not config: config = os_client_config.OpenStackConfig() try: @@ -89,10 +91,10 @@ def openstack_cloud(config=None, **kwargs): except keystoneauth1.exceptions.auth_plugins.NoMatchingPlugin as e: raise OpenStackCloudException( "Invalid cloud configuration: {exc}".format(exc=str(e))) - return OpenStackCloud(cloud_config=cloud_config) + return OpenStackCloud(cloud_config=cloud_config, strict=strict) -def operator_cloud(config=None, **kwargs): +def operator_cloud(config=None, strict=False, **kwargs): if 'interface' not in kwargs: kwargs['interface'] = 'admin' if not config: @@ -102,4 +104,4 @@ def operator_cloud(config=None, **kwargs): except keystoneauth1.exceptions.auth_plugins.NoMatchingPlugin as e: raise OpenStackCloudException( "Invalid cloud configuration: {exc}".format(exc=str(e))) - return OperatorCloud(cloud_config=cloud_config) + return OperatorCloud(cloud_config=cloud_config, strict=strict) diff --git a/shade/_normalize.py b/shade/_normalize.py index 24dbbdfaf..8a9561b39 100644 --- a/shade/_normalize.py +++ b/shade/_normalize.py @@ -12,8 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import ast - import munch import six @@ -57,10 +55,10 @@ _SERVER_FIELDS = ( def _to_bool(value): if isinstance(value, six.string_types): - # ast.literal_eval becomes VERY unhappy on empty strings if not value: return False - return ast.literal_eval(value.lower().capitalize()) + prospective = value.lower().capitalize() + return prospective == 'True' return bool(value) @@ -72,6 +70,13 @@ def _pop_float(resource, key): return float(resource.pop(key, 0) or 0) +def _pop_or_get(resource, key, default, strict): + if strict: + return resource.pop(key, default) + else: + return resource.get(key, default) + + class Normalizer(object): '''Mix-in class to provide the normalization functions. @@ -99,11 +104,14 @@ class Normalizer(object): flavor.pop('HUMAN_ID', None) flavor.pop('human_id', None) - ephemeral = int(flavor.pop('OS-FLV-EXT-DATA:ephemeral', 0)) + ephemeral = int(_pop_or_get( + flavor, 'OS-FLV-EXT-DATA:ephemeral', 0, self.strict_mode)) ephemeral = flavor.pop('ephemeral', ephemeral) - is_public = _to_bool(flavor.pop('os-flavor-access:is_public', True)) - is_public = _to_bool(flavor.pop('is_public', True)) - is_disabled = _to_bool(flavor.pop('OS-FLV-DISABLED:disabled', False)) + is_public = _to_bool(_pop_or_get( + flavor, 'os-flavor-access:is_public', True, self.strict_mode)) + is_public = _to_bool(flavor.pop('is_public', is_public)) + is_disabled = _to_bool(_pop_or_get( + flavor, 'OS-FLV-DISABLED:disabled', False, self.strict_mode)) extra_specs = flavor.pop('extra_specs', {}) new_flavor['location'] = self.current_location @@ -122,11 +130,9 @@ class Normalizer(object): new_flavor['extra_specs'] = extra_specs # Backwards compat with nova - passthrough values - for (k, v) in new_flavor['properties'].items(): - new_flavor.setdefault(k, v) - new_flavor['OS-FLV-DISABLED:disabled'] = is_disabled - new_flavor['OS-FLV-EXT-DATA:ephemeral'] = ephemeral - new_flavor['os-flavor-access:is_public'] = is_public + if not self.strict_mode: + for (k, v) in new_flavor['properties'].items(): + new_flavor.setdefault(k, v) return new_flavor @@ -164,9 +170,10 @@ class Normalizer(object): new_image['is_public'] = is_public # Backwards compat with glance - for key, val in properties.items(): - new_image[key] = val - new_image['protected'] = protected + if not self.strict_mode: + for key, val in properties.items(): + new_image[key] = val + new_image['protected'] = protected return new_image def _normalize_secgroups(self, groups): @@ -204,10 +211,11 @@ class Normalizer(object): ret['properties'] = group # Backwards compat with Neutron - ret['tenant_id'] = project_id - ret['project_id'] = project_id - for key, val in ret['properties'].items(): - ret.setdefault(key, val) + if not self.strict_mode: + ret['tenant_id'] = project_id + ret['project_id'] = project_id + for key, val in ret['properties'].items(): + ret.setdefault(key, val) return ret @@ -260,10 +268,11 @@ class Normalizer(object): ret['properties'] = rule # Backwards compat with Neutron - ret['tenant_id'] = project_id - ret['project_id'] = project_id - for key, val in ret['properties'].items(): - ret.setdefault(key, val) + if not self.strict_mode: + ret['tenant_id'] = project_id + ret['project_id'] = project_id + for key, val in ret['properties'].items(): + ret.setdefault(key, val) return ret def _normalize_servers(self, servers): @@ -299,12 +308,15 @@ class Normalizer(object): project_id = server.pop('tenant_id', '') project_id = server.pop('project_id', project_id) - az = server.get('OS-EXT-AZ:availability_zone', None) + az = _pop_or_get( + server, 'OS-EXT-AZ:availability_zone', None, self.strict_mode) ret['location'] = self._get_current_location( project_id=project_id, zone=az) # Ensure volumes is always in the server dict, even if empty - ret['volumes'] = [] + ret['volumes'] = _pop_or_get( + server, 'os-extended-volumes:volumes_attached', + [], self.strict_mode) config_drive = server.pop('config_drive', False) ret['has_config_drive'] = _to_bool(config_drive) @@ -315,7 +327,8 @@ class Normalizer(object): ret['progress'] = _pop_int(server, 'progress') # Leave these in so that the general properties handling works - ret['disk_config'] = server.get('OS-DCF:diskConfig') + ret['disk_config'] = _pop_or_get( + server, 'OS-DCF:diskConfig', None, self.strict_mode) for key in ( 'OS-EXT-STS:power_state', 'OS-EXT-STS:task_state', @@ -323,24 +336,25 @@ class Normalizer(object): 'OS-SRV-USG:launched_at', 'OS-SRV-USG:terminated_at'): short_key = key.split(':')[1] - ret[short_key] = server.get(key) + ret[short_key] = _pop_or_get(server, key, None, self.strict_mode) for field in _SERVER_FIELDS: ret[field] = server.pop(field, None) ret['interface_ip'] = '' ret['properties'] = server.copy() - for key, val in ret['properties'].items(): - ret.setdefault(key, val) # Backwards compat - ret['hostId'] = host_id - ret['config_drive'] = config_drive - ret['project_id'] = project_id - ret['tenant_id'] = project_id - ret['region'] = self.region_name - ret['cloud'] = self.name - ret['az'] = az + if not self.strict_mode: + ret['hostId'] = host_id + ret['config_drive'] = config_drive + ret['project_id'] = project_id + ret['tenant_id'] = project_id + ret['region'] = self.region_name + ret['cloud'] = self.name + ret['az'] = az + for key, val in ret['properties'].items(): + ret.setdefault(key, val) return ret def _normalize_floating_ips(self, ips): @@ -406,18 +420,22 @@ class Normalizer(object): attached=attached, fixed_ip_address=fixed_ip_address, floating_ip_address=floating_ip_address, - floating_network_id=network_id, id=id, location=self._get_current_location(project_id=project_id), network=network_id, - port_id=port_id, - project_id=project_id, - router_id=router_id, + port=port_id, + router=router_id, status=status, - tenant_id=project_id, properties=ip.copy(), ) - for key, val in ret['properties'].items(): - ret.setdefault(key, val) + # Backwards compat + if not self.strict_mode: + ret['port_id'] = port_id + ret['router_id'] = router_id + ret['project_id'] = project_id + ret['tenant_id'] = project_id + ret['floating_network_id'] = network_id, + for key, val in ret['properties'].items(): + ret.setdefault(key, val) return ret diff --git a/shade/openstackcloud.py b/shade/openstackcloud.py index ce8e6c1a6..ddcb8b24c 100644 --- a/shade/openstackcloud.py +++ b/shade/openstackcloud.py @@ -123,6 +123,8 @@ class OpenStackCloud(_normalize.Normalizer): have all of the wrapped exceptions be emitted to the error log. This flag will enable that behavior. + :param bool strict: Only return documented attributes for each resource + as per the shade Data Model contract. (Default False) :param CloudConfig cloud_config: Cloud config object from os-client-config In the future, this will be the only way to pass in cloud configuration, but is @@ -132,7 +134,9 @@ class OpenStackCloud(_normalize.Normalizer): def __init__( self, cloud_config=None, - manager=None, log_inner_exceptions=False, **kwargs): + manager=None, log_inner_exceptions=False, + strict=False, + **kwargs): if log_inner_exceptions: OpenStackCloudException.log_inner_exceptions = True @@ -151,6 +155,7 @@ class OpenStackCloud(_normalize.Normalizer): self.image_api_use_tasks = cloud_config.config['image_api_use_tasks'] self.secgroup_source = cloud_config.config['secgroup_source'] self.force_ipv4 = cloud_config.force_ipv4 + self.strict_mode = strict # Provide better error message for people with stale OCC if cloud_config.get_external_ipv4_networks is None: diff --git a/shade/tests/unit/base.py b/shade/tests/unit/base.py index 8e3c18b68..f731f78f8 100644 --- a/shade/tests/unit/base.py +++ b/shade/tests/unit/base.py @@ -75,6 +75,10 @@ class BaseTestCase(base.TestCase): self.cloud = shade.OpenStackCloud( cloud_config=self.cloud_config, log_inner_exceptions=True) + self.strict_cloud = shade.OpenStackCloud( + cloud_config=self.cloud_config, + log_inner_exceptions=True, + strict=True) self.op_cloud = shade.OperatorCloud( cloud_config=self.cloud_config, log_inner_exceptions=True) diff --git a/shade/tests/unit/test__utils.py b/shade/tests/unit/test__utils.py index 704e66e1b..4ca82d52a 100644 --- a/shade/tests/unit/test__utils.py +++ b/shade/tests/unit/test__utils.py @@ -12,7 +12,6 @@ # License for the specific language governing permissions and limitations # under the License. -import mock import testtools from shade import _utils @@ -80,131 +79,6 @@ class TestUtils(base.TestCase): }}) self.assertEqual([el2, el3], ret) - def test_normalize_secgroups(self): - nova_secgroup = dict( - id='abc123', - name='nova_secgroup', - description='A Nova security group', - rules=[ - dict(id='123', from_port=80, to_port=81, ip_protocol='tcp', - ip_range={'cidr': '0.0.0.0/0'}, parent_group_id='xyz123') - ] - ) - - expected = dict( - id='abc123', - name='nova_secgroup', - description='A Nova security group', - tenant_id='', - project_id='', - properties={}, - location=dict( - region_name='RegionOne', - zone=None, - project=dict( - domain_name=None, - id=mock.ANY, - domain_id=None, - name='admin'), - cloud='_test_cloud_'), - security_group_rules=[ - dict(id='123', direction='ingress', ethertype='IPv4', - port_range_min=80, port_range_max=81, protocol='tcp', - remote_ip_prefix='0.0.0.0/0', security_group_id='xyz123', - properties={}, - tenant_id='', - project_id='', - remote_group_id=None, - location=dict( - region_name='RegionOne', - zone=None, - project=dict( - domain_name=None, - id=mock.ANY, - domain_id=None, - name='admin'), - cloud='_test_cloud_')) - ] - ) - - retval = self.cloud._normalize_secgroup(nova_secgroup) - self.assertEqual(expected, retval) - - def test_normalize_secgroups_negone_port(self): - nova_secgroup = dict( - id='abc123', - name='nova_secgroup', - description='A Nova security group with -1 ports', - rules=[ - dict(id='123', from_port=-1, to_port=-1, ip_protocol='icmp', - ip_range={'cidr': '0.0.0.0/0'}, parent_group_id='xyz123') - ] - ) - - retval = self.cloud._normalize_secgroup(nova_secgroup) - self.assertIsNone(retval['security_group_rules'][0]['port_range_min']) - self.assertIsNone(retval['security_group_rules'][0]['port_range_max']) - - def test_normalize_secgroup_rules(self): - nova_rules = [ - dict(id='123', from_port=80, to_port=81, ip_protocol='tcp', - ip_range={'cidr': '0.0.0.0/0'}, parent_group_id='xyz123') - ] - expected = [ - dict(id='123', direction='ingress', ethertype='IPv4', - port_range_min=80, port_range_max=81, protocol='tcp', - remote_ip_prefix='0.0.0.0/0', security_group_id='xyz123', - tenant_id='', project_id='', remote_group_id=None, - properties={}, - location=dict( - region_name='RegionOne', - zone=None, - project=dict( - domain_name=None, - id=mock.ANY, - domain_id=None, - name='admin'), - cloud='_test_cloud_')) - ] - retval = self.cloud._normalize_secgroup_rules(nova_rules) - self.assertEqual(expected, retval) - - def test_normalize_volumes_v1(self): - vol = dict( - display_name='test', - display_description='description', - bootable=u'false', # unicode type - multiattach='true', # str type - ) - expected = dict( - name=vol['display_name'], - display_name=vol['display_name'], - description=vol['display_description'], - display_description=vol['display_description'], - bootable=False, - multiattach=True, - ) - retval = _utils.normalize_volumes([vol]) - self.assertEqual([expected], retval) - - def test_normalize_volumes_v2(self): - vol = dict( - display_name='test', - display_description='description', - bootable=False, - multiattach=True, - ) - expected = dict( - name=vol['display_name'], - display_name=vol['display_name'], - description=vol['display_description'], - display_description=vol['display_description'], - bootable=False, - multiattach=True, - ) - retval = _utils.normalize_volumes([vol]) - self.assertEqual([expected], retval) - def test_safe_dict_min_ints(self): """Test integer comparison""" data = [{'f1': 3}, {'f1': 2}, {'f1': 1}] diff --git a/shade/tests/unit/test_normalize.py b/shade/tests/unit/test_normalize.py new file mode 100644 index 000000000..a3840ac10 --- /dev/null +++ b/shade/tests/unit/test_normalize.py @@ -0,0 +1,394 @@ +# -*- coding: utf-8 -*- + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from shade import _utils +from shade.tests.unit import base + +RAW_SERVER_DICT = { + 'HUMAN_ID': True, + 'NAME_ATTR': 'name', + 'OS-DCF:diskConfig': u'MANUAL', + 'OS-EXT-AZ:availability_zone': u'ca-ymq-2', + 'OS-EXT-STS:power_state': 1, + 'OS-EXT-STS:task_state': None, + 'OS-EXT-STS:vm_state': u'active', + 'OS-SRV-USG:launched_at': u'2015-08-01T19:52:02.000000', + 'OS-SRV-USG:terminated_at': None, + 'accessIPv4': u'', + 'accessIPv6': u'', + 'addresses': { + u'public': [{ + u'OS-EXT-IPS-MAC:mac_addr': u'fa:16:3e:9f:46:3e', + u'OS-EXT-IPS:type': u'fixed', + u'addr': u'2604:e100:1:0:f816:3eff:fe9f:463e', + u'version': 6 + }, { + u'OS-EXT-IPS-MAC:mac_addr': u'fa:16:3e:9f:46:3e', + u'OS-EXT-IPS:type': u'fixed', + u'addr': u'162.253.54.192', + u'version': 4}]}, + 'config_drive': u'True', + 'created': u'2015-08-01T19:52:16Z', + 'flavor': { + u'id': u'bbcb7eb5-5c8d-498f-9d7e-307c575d3566', + u'links': [{ + u'href': u'https://compute-ca-ymq-1.vexxhost.net/db9/flavors/bbc', + u'rel': u'bookmark'}]}, + 'hostId': u'bd37', + 'human_id': u'mordred-irc', + 'id': u'811c5197-dba7-4d3a-a3f6-68ca5328b9a7', + 'image': { + u'id': u'69c99b45-cd53-49de-afdc-f24789eb8f83', + u'links': [{ + u'href': u'https://compute-ca-ymq-1.vexxhost.net/db9/images/69c', + u'rel': u'bookmark'}]}, + 'key_name': u'mordred', + 'links': [{ + u'href': u'https://compute-ca-ymq-1.vexxhost.net/v2/db9/servers/811', + u'rel': u'self' + }, { + u'href': u'https://compute-ca-ymq-1.vexxhost.net/db9/servers/811', + u'rel': u'bookmark'}], + 'metadata': {u'group': u'irc', u'groups': u'irc,enabled'}, + 'name': u'mordred-irc', + 'networks': {u'public': [u'2604:e100:1:0:f816:3eff:fe9f:463e', + u'162.253.54.192']}, + 'os-extended-volumes:volumes_attached': [], + 'progress': 0, + 'request_ids': [], + 'security_groups': [{u'name': u'default'}], + 'status': u'ACTIVE', + 'tenant_id': u'db92b20496ae4fbda850a689ea9d563f', + 'updated': u'2016-10-15T15:49:29Z', + 'user_id': u'e9b21dc437d149858faee0898fb08e92'} + + +class TestUtils(base.TestCase): + + def test_normalize_servers_strict(self): + raw_server = RAW_SERVER_DICT.copy() + expected = { + 'accessIPv4': u'', + 'accessIPv6': u'', + 'addresses': { + u'public': [{ + u'OS-EXT-IPS-MAC:mac_addr': u'fa:16:3e:9f:46:3e', + u'OS-EXT-IPS:type': u'fixed', + u'addr': u'2604:e100:1:0:f816:3eff:fe9f:463e', + u'version': 6 + }, { + u'OS-EXT-IPS-MAC:mac_addr': u'fa:16:3e:9f:46:3e', + u'OS-EXT-IPS:type': u'fixed', + u'addr': u'162.253.54.192', + u'version': 4}]}, + 'adminPass': None, + 'created': u'2015-08-01T19:52:16Z', + 'disk_config': u'MANUAL', + 'flavor': {u'id': u'bbcb7eb5-5c8d-498f-9d7e-307c575d3566'}, + 'has_config_drive': True, + 'host_id': u'bd37', + 'id': u'811c5197-dba7-4d3a-a3f6-68ca5328b9a7', + 'image': {u'id': u'69c99b45-cd53-49de-afdc-f24789eb8f83'}, + 'interface_ip': u'', + 'key_name': u'mordred', + 'launched_at': u'2015-08-01T19:52:02.000000', + 'location': { + 'cloud': '_test_cloud_', + 'project': { + 'domain_id': None, + 'domain_name': None, + 'id': u'db92b20496ae4fbda850a689ea9d563f', + 'name': None}, + 'region_name': u'RegionOne', + 'zone': u'ca-ymq-2'}, + 'metadata': {u'group': u'irc', u'groups': u'irc,enabled'}, + 'name': u'mordred-irc', + 'networks': { + u'public': [ + u'2604:e100:1:0:f816:3eff:fe9f:463e', + u'162.253.54.192']}, + 'power_state': 1, + 'private_v4': None, + 'progress': 0, + 'properties': { + 'request_ids': []}, + 'public_v4': None, + 'public_v6': None, + 'security_groups': [{u'name': u'default'}], + 'status': u'ACTIVE', + 'task_state': None, + 'terminated_at': None, + 'updated': u'2016-10-15T15:49:29Z', + 'user_id': u'e9b21dc437d149858faee0898fb08e92', + 'vm_state': u'active', + 'volumes': []} + retval = self.strict_cloud._normalize_server(raw_server).toDict() + self.assertEqual(expected, retval) + + def test_normalize_servers_normal(self): + raw_server = RAW_SERVER_DICT.copy() + expected = { + 'OS-DCF:diskConfig': u'MANUAL', + 'OS-EXT-AZ:availability_zone': u'ca-ymq-2', + 'OS-EXT-STS:power_state': 1, + 'OS-EXT-STS:task_state': None, + 'OS-EXT-STS:vm_state': u'active', + 'OS-SRV-USG:launched_at': u'2015-08-01T19:52:02.000000', + 'OS-SRV-USG:terminated_at': None, + 'accessIPv4': u'', + 'accessIPv6': u'', + 'addresses': { + u'public': [{ + u'OS-EXT-IPS-MAC:mac_addr': u'fa:16:3e:9f:46:3e', + u'OS-EXT-IPS:type': u'fixed', + u'addr': u'2604:e100:1:0:f816:3eff:fe9f:463e', + u'version': 6 + }, { + u'OS-EXT-IPS-MAC:mac_addr': u'fa:16:3e:9f:46:3e', + u'OS-EXT-IPS:type': u'fixed', + u'addr': u'162.253.54.192', + u'version': 4}]}, + 'adminPass': None, + 'az': u'ca-ymq-2', + 'cloud': '_test_cloud_', + 'config_drive': u'True', + 'created': u'2015-08-01T19:52:16Z', + 'disk_config': u'MANUAL', + 'flavor': {u'id': u'bbcb7eb5-5c8d-498f-9d7e-307c575d3566'}, + 'has_config_drive': True, + 'hostId': u'bd37', + 'host_id': u'bd37', + 'id': u'811c5197-dba7-4d3a-a3f6-68ca5328b9a7', + 'image': {u'id': u'69c99b45-cd53-49de-afdc-f24789eb8f83'}, + 'interface_ip': '', + 'key_name': u'mordred', + 'launched_at': u'2015-08-01T19:52:02.000000', + 'location': { + 'cloud': '_test_cloud_', + 'project': { + 'domain_id': None, + 'domain_name': None, + 'id': u'db92b20496ae4fbda850a689ea9d563f', + 'name': None}, + 'region_name': u'RegionOne', + 'zone': u'ca-ymq-2'}, + 'metadata': {u'group': u'irc', u'groups': u'irc,enabled'}, + 'name': u'mordred-irc', + 'networks': { + u'public': [ + u'2604:e100:1:0:f816:3eff:fe9f:463e', + u'162.253.54.192']}, + 'os-extended-volumes:volumes_attached': [], + 'power_state': 1, + 'private_v4': None, + 'progress': 0, + 'project_id': u'db92b20496ae4fbda850a689ea9d563f', + 'properties': { + 'OS-DCF:diskConfig': u'MANUAL', + 'OS-EXT-AZ:availability_zone': u'ca-ymq-2', + 'OS-EXT-STS:power_state': 1, + 'OS-EXT-STS:task_state': None, + 'OS-EXT-STS:vm_state': u'active', + 'OS-SRV-USG:launched_at': u'2015-08-01T19:52:02.000000', + 'OS-SRV-USG:terminated_at': None, + 'os-extended-volumes:volumes_attached': [], + 'request_ids': []}, + 'public_v4': None, + 'public_v6': None, + 'region': u'RegionOne', + 'request_ids': [], + 'security_groups': [{u'name': u'default'}], + 'status': u'ACTIVE', + 'task_state': None, + 'tenant_id': u'db92b20496ae4fbda850a689ea9d563f', + 'terminated_at': None, + 'updated': u'2016-10-15T15:49:29Z', + 'user_id': u'e9b21dc437d149858faee0898fb08e92', + 'vm_state': u'active', + 'volumes': []} + retval = self.cloud._normalize_server(raw_server).toDict() + self.assertEqual(expected, retval) + + def test_normalize_secgroups_strict(self): + nova_secgroup = dict( + id='abc123', + name='nova_secgroup', + description='A Nova security group', + rules=[ + dict(id='123', from_port=80, to_port=81, ip_protocol='tcp', + ip_range={'cidr': '0.0.0.0/0'}, parent_group_id='xyz123') + ] + ) + + expected = dict( + id='abc123', + name='nova_secgroup', + description='A Nova security group', + properties={}, + location=dict( + region_name='RegionOne', + zone=None, + project=dict( + domain_name=None, + id=mock.ANY, + domain_id=None, + name='admin'), + cloud='_test_cloud_'), + security_group_rules=[ + dict(id='123', direction='ingress', ethertype='IPv4', + port_range_min=80, port_range_max=81, protocol='tcp', + remote_ip_prefix='0.0.0.0/0', security_group_id='xyz123', + properties={}, + remote_group_id=None, + location=dict( + region_name='RegionOne', + zone=None, + project=dict( + domain_name=None, + id=mock.ANY, + domain_id=None, + name='admin'), + cloud='_test_cloud_')) + ] + ) + + retval = self.strict_cloud._normalize_secgroup(nova_secgroup) + self.assertEqual(expected, retval) + + def test_normalize_secgroups(self): + nova_secgroup = dict( + id='abc123', + name='nova_secgroup', + description='A Nova security group', + rules=[ + dict(id='123', from_port=80, to_port=81, ip_protocol='tcp', + ip_range={'cidr': '0.0.0.0/0'}, parent_group_id='xyz123') + ] + ) + + expected = dict( + id='abc123', + name='nova_secgroup', + description='A Nova security group', + tenant_id='', + project_id='', + properties={}, + location=dict( + region_name='RegionOne', + zone=None, + project=dict( + domain_name=None, + id=mock.ANY, + domain_id=None, + name='admin'), + cloud='_test_cloud_'), + security_group_rules=[ + dict(id='123', direction='ingress', ethertype='IPv4', + port_range_min=80, port_range_max=81, protocol='tcp', + remote_ip_prefix='0.0.0.0/0', security_group_id='xyz123', + properties={}, + tenant_id='', + project_id='', + remote_group_id=None, + location=dict( + region_name='RegionOne', + zone=None, + project=dict( + domain_name=None, + id=mock.ANY, + domain_id=None, + name='admin'), + cloud='_test_cloud_')) + ] + ) + + retval = self.cloud._normalize_secgroup(nova_secgroup) + self.assertEqual(expected, retval) + + def test_normalize_secgroups_negone_port(self): + nova_secgroup = dict( + id='abc123', + name='nova_secgroup', + description='A Nova security group with -1 ports', + rules=[ + dict(id='123', from_port=-1, to_port=-1, ip_protocol='icmp', + ip_range={'cidr': '0.0.0.0/0'}, parent_group_id='xyz123') + ] + ) + + retval = self.cloud._normalize_secgroup(nova_secgroup) + self.assertIsNone(retval['security_group_rules'][0]['port_range_min']) + self.assertIsNone(retval['security_group_rules'][0]['port_range_max']) + + def test_normalize_secgroup_rules(self): + nova_rules = [ + dict(id='123', from_port=80, to_port=81, ip_protocol='tcp', + ip_range={'cidr': '0.0.0.0/0'}, parent_group_id='xyz123') + ] + expected = [ + dict(id='123', direction='ingress', ethertype='IPv4', + port_range_min=80, port_range_max=81, protocol='tcp', + remote_ip_prefix='0.0.0.0/0', security_group_id='xyz123', + tenant_id='', project_id='', remote_group_id=None, + properties={}, + location=dict( + region_name='RegionOne', + zone=None, + project=dict( + domain_name=None, + id=mock.ANY, + domain_id=None, + name='admin'), + cloud='_test_cloud_')) + ] + retval = self.cloud._normalize_secgroup_rules(nova_rules) + self.assertEqual(expected, retval) + + def test_normalize_volumes_v1(self): + vol = dict( + display_name='test', + display_description='description', + bootable=u'false', # unicode type + multiattach='true', # str type + ) + expected = dict( + name=vol['display_name'], + display_name=vol['display_name'], + description=vol['display_description'], + display_description=vol['display_description'], + bootable=False, + multiattach=True, + ) + retval = _utils.normalize_volumes([vol]) + self.assertEqual([expected], retval) + + def test_normalize_volumes_v2(self): + vol = dict( + display_name='test', + display_description='description', + bootable=False, + multiattach=True, + ) + expected = dict( + name=vol['display_name'], + display_name=vol['display_name'], + description=vol['display_description'], + display_description=vol['display_description'], + bootable=False, + multiattach=True, + ) + retval = _utils.normalize_volumes([vol]) + self.assertEqual([expected], retval)