Avoid neutron calls at recovering precreated ports

This patch ensures only 1 get calls is made to neutron to get the
ports when recovering precreated ports upon controller reboot. It
also ensures only one call is made to get the subnet information
per subnet, instead of 1 per port. Further improvements can be
made but this at least reduces the pressure on neutron side.

Partially Implements blueprint ports-pool

Change-Id: I635e7d3ac1a76b014dcfb1ef7539cfd56e73dddd
This commit is contained in:
Luis Tomas Bolivar 2017-10-06 15:09:00 +00:00
parent bc8be6506d
commit 79d8e23de7
2 changed files with 301 additions and 137 deletions

View File

@ -425,6 +425,66 @@ class NestedVIFPool(BaseVIFPool):
parent_port = neutron.show_port(port_id).get('port')
return parent_port['fixed_ips'][0]['ip_address']
def _get_trunks_info(self):
"""Returns information about trunks and their subports.
This method searches for parent ports and subports among the active
neutron ports.
To find the parent ports it filters the ones that have trunk_details,
i.e., the ones that are the parent port of a trunk.
To find the subports to recover, it filters out the ports that are
already in used by running kubernetes pods. It also filters out the
ports whose device_owner is not related to subports, i.e., the ports
that are not attached to trunks, such as active ports allocated to
running VMs.
At the same time it collects information about ports subnets to
minimize the number of interaction with Neutron API.
It returns three dictionaries with the needed information about the
parent ports, subports and subnets
:return: 3 dicts with the trunk details (Key: trunk_id; Value: dict
containing ip and subports), subport details (Key: port_id; Value:
port_object), and subnet details (Key: subnet_id; Value: subnet dict)
"""
# REVISIT(ltomasbo): there is no need to recover the subports
# belonging to trunk ports whose parent port is DOWN as that means no
# pods can be scheduled there. We may need to update this if we allow
# lively extending the kubernetes cluster with VMs that already have
# precreated subports. For instance by shutting down and up a
# kubernetes Worker VM with subports already attached, and the
# controller is restarted in between.
parent_ports = {}
subports = {}
subnets = {}
all_active_ports = self._get_ports_by_attrs(status='ACTIVE')
in_use_ports = self._get_in_use_ports()
for port in all_active_ports:
trunk_details = port.get('trunk_details')
# Parent port
if trunk_details:
parent_ports[trunk_details['trunk_id']] = {
'ip': port['fixed_ips'][0]['ip_address'],
'subports': trunk_details['sub_ports']}
else:
# Filter to only get subports that are not in use
if (port['id'] not in in_use_ports and
port['device_owner'] in ['trunk:subport',
kl_const.DEVICE_OWNER]):
subports[port['id']] = port
# NOTE(ltomasbo): _get_subnet can be costly as it
# needs to call neutron to get network and subnet
# information. This ensures it is only called once
# per subnet in use
subnet_id = port['fixed_ips'][0]['subnet_id']
if not subnets.get(subnet_id):
subnets[subnet_id] = {subnet_id:
default_subnet._get_subnet(
subnet_id)}
return parent_ports, subports, subnets
def _recover_precreated_ports(self):
self._precreated_ports(action='recover')
LOG.info("PORTS POOL: pools updated with pre-created ports")
@ -447,75 +507,55 @@ class NestedVIFPool(BaseVIFPool):
# when a port is attached to a trunk. However, that is not the case
# for other ML2 drivers, such as ODL. So we also need to look for
# compute:kuryr
if config.CONF.kubernetes.port_debug:
available_ports = self._get_ports_by_attrs(
name=constants.KURYR_PORT_NAME, device_owner=[
'trunk:subport', kl_const.DEVICE_OWNER])
else:
kuryr_subports = self._get_ports_by_attrs(
device_owner=['trunk:subport', kl_const.DEVICE_OWNER])
in_use_ports = self._get_in_use_ports()
available_ports = [subport for subport in kuryr_subports
if subport['id'] not in in_use_ports]
if not available_ports:
parent_ports, available_subports, subnets = self._get_trunks_info()
if not available_subports:
return
trunk_ports = neutron.list_trunks().get('trunks')
for trunk in trunk_ports:
try:
host_addr = self._get_parent_port_ip(trunk['port_id'])
except n_exc.PortNotFoundClient:
LOG.debug('Unable to find parent port for trunk port %s.',
trunk['port_id'])
continue
for trunk_id, parent_port in parent_ports.items():
host_addr = parent_port.get('ip')
if trunk_ips and host_addr not in trunk_ips:
continue
for subport in trunk.get('sub_ports'):
kuryr_subport = None
for port in available_ports:
if port['id'] == subport['port_id']:
kuryr_subport = port
break
for subport in parent_port.get('subports'):
kuryr_subport = available_subports.get(subport['port_id'])
if kuryr_subport:
pool_key = (host_addr, kuryr_subport['project_id'],
tuple(kuryr_subport['security_groups']))
if action == 'recover':
subnet_id = kuryr_subport['fixed_ips'][0]['subnet_id']
subnet = {
subnet_id: default_subnet._get_subnet(subnet_id)}
subnet = subnets[subnet_id]
vif = ovu.neutron_to_osvif_vif_nested_vlan(
kuryr_subport, subnet, subport['segmentation_id'])
self._existing_vifs[subport['port_id']] = vif
self._existing_vifs[kuryr_subport['id']] = vif
self._available_ports_pools.setdefault(
pool_key, []).append(subport['port_id'])
pool_key, []).append(kuryr_subport['id'])
elif action == 'free':
try:
self._drv_vif._remove_subport(neutron, trunk['id'],
subport['port_id'])
neutron.delete_port(subport['port_id'])
self._drv_vif._remove_subport(neutron, trunk_id,
kuryr_subport['id'])
neutron.delete_port(kuryr_subport['id'])
self._drv_vif._release_vlan_id(
subport['segmentation_id'])
del self._existing_vifs[subport['port_id']]
del self._existing_vifs[kuryr_subport['id']]
self._available_ports_pools[pool_key].remove(
subport['port_id'])
kuryr_subport['id'])
except n_exc.PortNotFoundClient:
LOG.debug('Unable to release port %s as it no '
'longer exists.', subport['port_id'])
'longer exists.', kuryr_subport['id'])
except KeyError:
LOG.debug('Port %s is not in the ports list.',
subport['port_id'])
kuryr_subport['id'])
except n_exc.NeutronClientException:
LOG.warning('Error removing the subport %s',
subport['port_id'])
kuryr_subport['id'])
except ValueError:
LOG.debug('Port %s is not in the available ports '
'pool.', subport['port_id'])
'pool.', kuryr_subport['id'])
def force_populate_pool(self, trunk_ip, project_id, subnets,
security_groups, num_ports):

View File

@ -662,7 +662,7 @@ class NeutronVIFPool(test_base.TestCase):
@ddt.ddt
class NestedVIFPool(test_base.TestCase):
def _get_trunk_obj(self, port_id=None, subport_id=None):
def _get_trunk_obj(self, port_id=None, subport_id=None, trunk_id=None):
trunk_obj = {
'status': 'ACTIVE',
'name': 'trunk-01aa31ea-5adf-4776-9c5d-21b50dba0ccc',
@ -684,9 +684,19 @@ class NestedVIFPool(test_base.TestCase):
trunk_obj['port_id'] = port_id
if subport_id:
trunk_obj['sub_ports'][0]['port_id'] = subport_id
if trunk_id:
trunk_obj['id'] = trunk_id
return trunk_obj
def _get_parent_ports(self, trunk_objs):
parent_ports = {}
for trunk_obj in trunk_objs:
parent_ports[trunk_obj['id']] = {
'ip': 'kuryr-devstack',
'subports': trunk_obj['sub_ports']}
return parent_ports
@mock.patch('eventlet.spawn')
def test__get_port_from_pool(self, m_eventlet):
cls = vif_pool.NestedVIFPool
@ -1010,31 +1020,110 @@ class NestedVIFPool(test_base.TestCase):
self.assertEqual(ip_address, cls._get_parent_port_ip(m_driver,
port_id))
@mock.patch('kuryr_kubernetes.os_vif_util.'
'neutron_to_osvif_vif_nested_vlan')
@mock.patch('kuryr_kubernetes.controller.drivers.default_subnet.'
'_get_subnet')
def test__precreated_ports_recover(self, m_get_subnet, m_to_osvif):
def test__get_trunk_info(self, m_get_subnet):
cls = vif_pool.NestedVIFPool
m_driver = mock.MagicMock(spec=cls)
port_id = mock.sentinel.port_id
trunk_port = get_port_obj(port_id=port_id)
trunk_id = mock.sentinel.id
trunk_details = {
'trunk_id': trunk_id,
'sub_ports': [{
'port_id': '85104e7d-8597-4bf7-94e7-a447ef0b50f1',
'segmentation_type': 'vlan',
'segmentation_id': 4056}]}
trunk_port['trunk_details'] = trunk_details
subport_id = mock.sentinel.subport_id
subport = get_port_obj(port_id=subport_id,
device_owner='trunk:subport')
m_driver._get_ports_by_attrs.return_value = [trunk_port, subport]
m_driver._get_in_use_ports.return_value = []
subnet = mock.sentinel.subnet
m_get_subnet.return_value = subnet
exp_p_ports = {trunk_id: {
'ip': trunk_port['fixed_ips'][0]['ip_address'],
'subports': trunk_details['sub_ports']}}
exp_subnets = {subport['fixed_ips'][0]['subnet_id']:
{subport['fixed_ips'][0]['subnet_id']: subnet}}
r_p_ports, r_subports, r_subnets = cls._get_trunks_info(m_driver)
self.assertEqual(r_p_ports, exp_p_ports)
self.assertEqual(r_subports, {subport_id: subport})
self.assertEqual(r_subnets, exp_subnets)
def test__get_trunk_info_empty(self):
cls = vif_pool.NestedVIFPool
m_driver = mock.MagicMock(spec=cls)
m_driver._get_ports_by_attrs.return_value = []
m_driver._get_in_use_ports.return_value = []
r_p_ports, r_subports, r_subnets = cls._get_trunks_info(m_driver)
self.assertEqual(r_p_ports, {})
self.assertEqual(r_subports, {})
self.assertEqual(r_subnets, {})
def test__get_trunk_info_no_trunk_details(self):
cls = vif_pool.NestedVIFPool
m_driver = mock.MagicMock(spec=cls)
port_id = mock.sentinel.port_id
port = get_port_obj(port_id=port_id)
port = get_port_obj(port_id=port_id, device_owner='compute:nova')
m_driver._get_ports_by_attrs.return_value = [port]
m_driver._get_in_use_ports.return_value = []
r_p_ports, r_subports, r_subnets = cls._get_trunks_info(m_driver)
self.assertEqual(r_p_ports, {})
self.assertEqual(r_subports, {})
self.assertEqual(r_subnets, {})
@mock.patch('kuryr_kubernetes.os_vif_util.'
'neutron_to_osvif_vif_nested_vlan')
def test__precreated_ports_recover(self, m_to_osvif):
cls = vif_pool.NestedVIFPool
m_driver = mock.MagicMock(spec=cls)
neutron = self.useFixture(k_fix.MockNeutronClient()).client
port_id = mock.sentinel.port_id
host_addr = mock.sentinel.host_addr
m_driver._available_ports_pools = {}
m_driver._existing_vifs = {}
m_driver._get_ports_by_attrs.side_effect = [[get_port_obj(
port_id=port_id, device_owner='trunk:subport')], []]
oslo_cfg.CONF.set_override('port_debug',
True,
group='kubernetes')
port_id = mock.sentinel.port_id
trunk_id = mock.sentinel.trunk_id
trunk_obj = self._get_trunk_obj(port_id=trunk_id, subport_id=port_id)
neutron.list_trunks.return_value = {'trunks': [trunk_obj]}
m_driver._get_parent_port_ip.return_value = host_addr
port = get_port_obj(port_id=port_id, device_owner='trunk:subport')
m_get_subnet.return_value = mock.sentinel.subnet
m_to_osvif.return_value = mock.sentinel.vif
p_ports = self._get_parent_ports([trunk_obj])
a_subports = {port_id: port}
subnet_id = port['fixed_ips'][0]['subnet_id']
subnet = mock.sentinel.subnet
subnets = {subnet_id: {subnet_id: subnet}}
m_driver._get_trunks_info.return_value = (p_ports, a_subports,
subnets)
vif = mock.sentinel.vif
m_to_osvif.return_value = vif
cls._precreated_ports(m_driver, 'recover')
neutron.list_trunks.assert_called_once()
m_driver._get_parent_port_ip.assert_called_with(trunk_id)
m_driver._get_trunks_info.assert_called_once()
self.assertEqual(m_driver._existing_vifs[port_id], vif)
pool_key = (port['binding:host_id'], port['project_id'],
tuple(port['security_groups']))
self.assertEqual(m_driver._available_ports_pools[pool_key], [port_id])
neutron.delete_port.assert_not_called()
def test__precreated_ports_free(self):
cls = vif_pool.NestedVIFPool
@ -1044,112 +1133,132 @@ class NestedVIFPool(test_base.TestCase):
vif_driver = mock.MagicMock(spec=cls_vif_driver)
m_driver._drv_vif = vif_driver
port_id = mock.sentinel.port_id
host_addr = mock.sentinel.host_addr
oslo_cfg.CONF.set_override('port_debug',
True,
group='kubernetes')
subport_obj = get_port_obj(port_id=port_id,
device_owner='trunk:subport')
m_driver._get_ports_by_attrs.side_effect = [[subport_obj], []]
port_id = mock.sentinel.port_id
trunk_id = mock.sentinel.trunk_id
trunk_obj = self._get_trunk_obj(port_id=trunk_id, subport_id=port_id)
pool_key = (host_addr, subport_obj['id'],
tuple(subport_obj['security_groups']))
m_driver._available_ports_pools = {pool_key: port_id}
port = get_port_obj(port_id=port_id, device_owner='trunk:subport')
neutron.list_trunks.return_value = {'trunks': [trunk_obj]}
m_driver._get_parent_port_ip.return_value = host_addr
p_ports = self._get_parent_ports([trunk_obj])
a_subports = {port_id: port}
subnet_id = port['fixed_ips'][0]['subnet_id']
subnet = mock.sentinel.subnet
subnets = {subnet_id: {subnet_id: subnet}}
m_driver._get_trunks_info.return_value = (p_ports, a_subports,
subnets)
pool_key = (port['binding:host_id'], port['project_id'],
tuple(port['security_groups']))
m_driver._available_ports_pools = {pool_key: [port_id]}
m_driver._existing_vifs = {port_id: mock.sentinel.vif}
cls._precreated_ports(m_driver, 'free')
neutron.list_trunks.assert_called_once()
m_driver._get_parent_port_ip.assert_called_with(trunk_id)
m_driver._get_trunks_info.assert_called_once()
m_driver._drv_vif._remove_subport.assert_called_once()
neutron.delete_port.assert_called_once()
m_driver._drv_vif._release_vlan_id.assert_called_once()
self.assertEqual(m_driver._existing_vifs, {})
self.assertEqual(m_driver._available_ports_pools[pool_key], [])
@mock.patch('kuryr_kubernetes.os_vif_util.'
'neutron_to_osvif_vif_nested_vlan')
@mock.patch('kuryr_kubernetes.controller.drivers.default_subnet.'
'_get_subnet')
def test__precreated_ports_recover_several_trunks(self, m_get_subnet,
m_to_osvif):
def test__precreated_ports_recover_several_trunks(self, m_to_osvif):
cls = vif_pool.NestedVIFPool
m_driver = mock.MagicMock(spec=cls)
neutron = self.useFixture(k_fix.MockNeutronClient()).client
m_driver._available_ports_pools = {}
m_driver._existing_vifs = {}
oslo_cfg.CONF.set_override('port_debug',
True,
group='kubernetes')
port_id1 = mock.sentinel.port_id1
host_addr1 = mock.sentinel.host_addr1
trunk_id1 = mock.sentinel.trunk_id1
port_id2 = mock.sentinel.port_id2
host_addr2 = mock.sentinel.host_addr2
trunk_id2 = mock.sentinel.trunk_id2
port1 = get_port_obj(port_id=port_id1, device_owner='trunk:subport')
port2 = get_port_obj(port_id=port_id2, device_owner='trunk:subport')
m_driver._get_ports_by_attrs.side_effect = [[port1, port2], []]
trunk_obj1 = self._get_trunk_obj(port_id=trunk_id1,
subport_id=port_id1)
trunk_obj2 = self._get_trunk_obj(port_id=trunk_id2,
subport_id=port_id2)
neutron.list_trunks.return_value = {'trunks': [trunk_obj1,
trunk_obj2]}
m_driver._get_parent_port_ip.side_effect = [host_addr1, host_addr2]
subport_id=port_id2,
trunk_id=mock.sentinel.id)
port1 = get_port_obj(port_id=port_id1, device_owner='trunk:subport')
port2 = get_port_obj(port_id=port_id2, device_owner='trunk:subport')
p_ports = self._get_parent_ports([trunk_obj1, trunk_obj2])
a_subports = {port_id1: port1, port_id2: port2}
subnet_id = port1['fixed_ips'][0]['subnet_id']
subnet = mock.sentinel.subnet
m_get_subnet.return_value = subnet
m_to_osvif.return_value = mock.sentinel.vif
subnets = {subnet_id: {subnet_id: subnet}}
m_driver._get_trunks_info.return_value = (p_ports, a_subports,
subnets)
vif = mock.sentinel.vif
m_to_osvif.return_value = vif
cls._precreated_ports(m_driver, 'recover')
neutron.list_trunks.asser_called_once()
m_driver._get_parent_port_ip.assert_has_calls([mock.call(trunk_id1),
mock.call(trunk_id2)])
calls = [mock.call(port1, {port1['fixed_ips'][0]['subnet_id']: subnet},
trunk_obj1['sub_ports'][0]['segmentation_id']),
mock.call(port2, {port2['fixed_ips'][0]['subnet_id']: subnet},
trunk_obj2['sub_ports'][0]['segmentation_id'])]
m_to_osvif.assert_has_calls(calls)
m_driver._get_trunks_info.assert_called_once()
self.assertEqual(m_driver._existing_vifs, {port_id1: vif,
port_id2: vif})
neutron.delete_port.assert_not_called()
@mock.patch('kuryr_kubernetes.os_vif_util.'
'neutron_to_osvif_vif_nested_vlan')
@mock.patch('kuryr_kubernetes.controller.drivers.default_subnet.'
'_get_subnet')
def test__precreated_ports_recover_several_subports(self, m_get_subnet,
m_to_osvif):
def test__precreated_ports_recover_several_subports(self, m_to_osvif):
cls = vif_pool.NestedVIFPool
m_driver = mock.MagicMock(spec=cls)
neutron = self.useFixture(k_fix.MockNeutronClient()).client
m_driver._available_ports_pools = {}
m_driver._existing_vifs = {}
oslo_cfg.CONF.set_override('port_debug',
True,
group='kubernetes')
port_id1 = mock.sentinel.port_id1
host_addr = mock.sentinel.host_addr
trunk_id = mock.sentinel.trunk_id
port_id2 = mock.sentinel.port_id2
port1 = get_port_obj(port_id=port_id1, device_owner='trunk:subport')
port2 = get_port_obj(port_id=port_id2, device_owner='trunk:subport')
m_driver._get_ports_by_attrs.side_effect = [[port1, port2], []]
trunk_id = mock.sentinel.trunk_id
trunk_obj = self._get_trunk_obj(port_id=trunk_id,
subport_id=port_id1)
trunk_obj['sub_ports'].append({'port_id': port_id2,
'segmentation_type': 'vlan',
'segmentation_id': 101})
neutron.list_trunks.return_value = {'trunks': [trunk_obj]}
m_driver._get_parent_port_ip.return_value = [host_addr]
port1 = get_port_obj(port_id=port_id1, device_owner='trunk:subport')
port2 = get_port_obj(port_id=port_id2, device_owner='trunk:subport')
p_ports = self._get_parent_ports([trunk_obj])
a_subports = {port_id1: port1, port_id2: port2}
subnet_id = port1['fixed_ips'][0]['subnet_id']
subnet = mock.sentinel.subnet
m_get_subnet.return_value = subnet
m_to_osvif.return_value = mock.sentinel.vif
subnets = {subnet_id: {subnet_id: subnet}}
m_driver._get_trunks_info.return_value = (p_ports, a_subports,
subnets)
vif = mock.sentinel.vif
m_to_osvif.return_value = vif
cls._precreated_ports(m_driver, 'recover')
neutron.list_trunks.asser_called_once()
m_driver._get_parent_port_ip.assert_called_once_with(trunk_id)
calls = [mock.call(port1, {port1['fixed_ips'][0]['subnet_id']: subnet},
trunk_obj['sub_ports'][0]['segmentation_id']),
mock.call(port2, {port2['fixed_ips'][0]['subnet_id']: subnet},
trunk_obj['sub_ports'][1]['segmentation_id'])]
m_to_osvif.assert_has_calls(calls)
m_driver._get_trunks_info.assert_called_once()
self.assertEqual(m_driver._existing_vifs, {port_id1: vif,
port_id2: vif})
pool_key = (port1['binding:host_id'], port1['project_id'],
tuple(port1['security_groups']))
self.assertEqual(m_driver._available_ports_pools[pool_key],
[port_id1, port_id2])
neutron.delete_port.assert_not_called()
@ddt.data(('recover'), ('free'))
def test__precreated_ports_no_ports(self, m_action):
@ -1157,10 +1266,28 @@ class NestedVIFPool(test_base.TestCase):
m_driver = mock.MagicMock(spec=cls)
neutron = self.useFixture(k_fix.MockNeutronClient()).client
m_driver._get_ports_by_attrs.return_value = []
oslo_cfg.CONF.set_override('port_debug',
True,
group='kubernetes')
m_driver._available_ports_pools = {}
m_driver._existing_vifs = {}
port_id = mock.sentinel.port_id
trunk_id = mock.sentinel.trunk_id
trunk_obj = self._get_trunk_obj(port_id=trunk_id, subport_id=port_id)
p_ports = self._get_parent_ports([trunk_obj])
a_subports = {}
subnets = {}
m_driver._get_trunks_info.return_value = (p_ports, a_subports,
subnets)
cls._precreated_ports(m_driver, m_action)
neutron.list_trunks.assert_not_called()
m_driver._get_trunks_info.assert_called_once()
self.assertEqual(m_driver._existing_vifs, {})
self.assertEqual(m_driver._available_ports_pools, {})
neutron.delete_port.assert_not_called()
@ddt.data(('recover'), ('free'))
def test__precreated_ports_no_trunks(self, m_action):
@ -1168,28 +1295,25 @@ class NestedVIFPool(test_base.TestCase):
m_driver = mock.MagicMock(spec=cls)
neutron = self.useFixture(k_fix.MockNeutronClient()).client
m_driver._get_ports_by_attrs.side_effect = [[get_port_obj(
device_owner='trunk:subport')], []]
neutron.list_trunks.return_value = {'trunks': []}
cls._precreated_ports(m_driver, m_action)
neutron.list_trunks.assert_called()
m_driver._get_parent_port_ip.assert_not_called()
@ddt.data(('recover'), ('free'))
def test__precreated_ports_exception(self, m_action):
cls = vif_pool.NestedVIFPool
m_driver = mock.MagicMock(spec=cls)
neutron = self.useFixture(k_fix.MockNeutronClient()).client
m_driver._available_ports_pools = {}
m_driver._existing_vifs = {}
oslo_cfg.CONF.set_override('port_debug',
True,
group='kubernetes')
port_id = mock.sentinel.port_id
m_driver._get_ports_by_attrs.side_effect = [[get_port_obj(
port_id=port_id, device_owner='trunk:subport')], []]
trunk_id = mock.sentinel.trunk_id
trunk_obj = self._get_trunk_obj(port_id=trunk_id)
neutron.list_trunks.return_value = {'trunks': [trunk_obj]}
m_driver._get_parent_port_ip.side_effect = n_exc.PortNotFoundClient
port = get_port_obj(port_id=port_id, device_owner='trunk:subport')
self.assertIsNone(cls._precreated_ports(m_driver, m_action))
neutron.list_trunks.assert_called()
m_driver._get_parent_port_ip.assert_called_with(trunk_id)
p_ports = {}
a_subports = {port_id: port}
subnet_id = port['fixed_ips'][0]['subnet_id']
subnet = mock.sentinel.subnet
subnets = {subnet_id: {subnet_id: subnet}}
m_driver._get_trunks_info.return_value = (p_ports, a_subports,
subnets)
cls._precreated_ports(m_driver, m_action)
m_driver._get_trunks_info.assert_called_once()
self.assertEqual(m_driver._existing_vifs, {})
self.assertEqual(m_driver._available_ports_pools, {})
neutron.delete_port.assert_not_called()