x
/
vmware-nsx
Code Issues Proposed changes

NSXv: Subnet create and attachment concurrency 

Address subnet create and VDR attachement concurrency issues.

Change-Id: Iece2c486c3b2006c6ccbe0acb18a93d208753f90
This commit is contained in:
Kobi Samoray 2017-01-18 11:35:10 +02:00
parent b7b7f9f685
commit 359ad6a8bf
10 changed files with 112 additions and 131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
vmware_nsx/plugins/nsx_v/drivers/distributed_router_driver.py
View File

@ -108,8 +108,8 @@ class RouterDistributedDriver(router_driver.RouterBaseDriver):
router_id)):
self.plugin._update_subnets_and_dnat_firewall(context,
router_db)
md_gw_data = self._get_metadata_gw_data(context, router_id)
self._update_routes(context, router_id, nexthop, md_gw_data)
md_gw_data = self._get_metadata_gw_data(context, router_id)
self._update_routes(context, router_id, nexthop, md_gw_data)
if 'admin_state_up' in r:
self.plugin._update_router_admin_state(
context, router_id, self.get_type(), r['admin_state_up'])
@ -176,11 +176,13 @@ class RouterDistributedDriver(router_driver.RouterBaseDriver):
context, router))
plr_id = self.edge_manager.get_plr_by_tlr_id(context, router_id)
tlr_edge_id = self._get_edge_id(context, router_id)
if not new_ext_net_id:
if plr_id:
# delete all plr relative conf
self.edge_manager.delete_plr_by_tlr_id(
context, plr_id, router_id)
with locking.LockManager.get_lock(tlr_edge_id):
self.edge_manager.delete_plr_by_tlr_id(
context, plr_id, router_id)
else:
# Connecting plr to the tlr if new_ext_net_id is not None.
if not plr_id:
@ -188,18 +190,22 @@ class RouterDistributedDriver(router_driver.RouterBaseDriver):
# retrieved by +get_router does not contain this information
availability_zone = self.get_router_az_by_id(
context, router['id'])
plr_id = self.edge_manager.create_plr_with_tlr_id(
context, router_id, router.get('name'), availability_zone)
with locking.LockManager.get_lock(tlr_edge_id):
plr_id = self.edge_manager.create_plr_with_tlr_id(
context, router_id, router.get('name'),
availability_zone)
if new_ext_net_id != org_ext_net_id and orgnexthop:
# network changed, so need to remove default gateway
# and all static routes before vnic can be configured
edge_utils.clear_gateway(self.nsx_v, context, plr_id)
with locking.LockManager.get_lock(tlr_edge_id):
edge_utils.clear_gateway(self.nsx_v, context, plr_id)
# Update external vnic if addr or mask is changed
if orgaddr != newaddr or orgmask != newmask:
self.edge_manager.update_external_interface(
self.nsx_v, context, plr_id,
new_ext_net_id, newaddr, newmask)
with locking.LockManager.get_lock(tlr_edge_id):
self.edge_manager.update_external_interface(
self.nsx_v, context, plr_id,
new_ext_net_id, newaddr, newmask)
# Update SNAT rules if ext net changed
# or ext net not changed but snat is changed.
@ -216,8 +222,9 @@ class RouterDistributedDriver(router_driver.RouterBaseDriver):
context, router, router_id=plr_id)
# update static routes in all
md_gw_data = self._get_metadata_gw_data(context, router_id)
self._update_routes(context, router_id, newnexthop, md_gw_data)
with locking.LockManager.get_lock(tlr_edge_id):
md_gw_data = self._get_metadata_gw_data(context, router_id)
self._update_routes(context, router_id, newnexthop, md_gw_data)
def _validate_multiple_subnets_routers(self, context, router_id,
interface_info):
@ -473,25 +480,26 @@ class RouterDistributedDriver(router_driver.RouterBaseDriver):
router_id)
# Reattach to regular DHCP Edge
dhcp_id = self.edge_manager.create_dhcp_edge_service(
context, network_id, subnet)
with locking.LockManager.get_lock(network_id):
dhcp_id = self.edge_manager.create_dhcp_edge_service(
context, network_id, subnet)
address_groups = self.plugin._create_network_dhcp_address_group(
context, network_id)
self.edge_manager.update_dhcp_edge_service(
context, network_id, address_groups=address_groups)
if dhcp_id:
edge_id = self.plugin._get_edge_id_by_rtr_id(context,
dhcp_id)
if edge_id:
with locking.LockManager.get_lock(str(edge_id)):
md_proxy_handler = (
self.plugin.metadata_proxy_handler)
if md_proxy_handler:
md_proxy_handler.configure_router_edge(
context, dhcp_id)
self.plugin.setup_dhcp_edge_fw_rules(
context, self.plugin, dhcp_id)
address_groups = self.plugin._create_network_dhcp_address_group(
context, network_id)
self.edge_manager.update_dhcp_edge_service(
context, network_id, address_groups=address_groups)
if dhcp_id:
edge_id = self.plugin._get_edge_id_by_rtr_id(context,
dhcp_id)
if edge_id:
with locking.LockManager.get_lock(str(edge_id)):
md_proxy_handler = (
self.plugin.metadata_proxy_handler)
if md_proxy_handler:
md_proxy_handler.configure_router_edge(
context, dhcp_id)
self.plugin.setup_dhcp_edge_fw_rules(
context, self.plugin, dhcp_id)
def _update_edge_router(self, context, router_id):
router = self.plugin._get_router(context.elevated(), router_id)

13
vmware_nsx/plugins/nsx_v/drivers/exclusive_router_driver.py
View File

@ -115,12 +115,13 @@ class RouterExclusiveDriver(router_driver.RouterBaseDriver):
intf_net_ids = (
self.plugin._get_internal_network_ids_by_router(context,
router_id))
for network_id in intf_net_ids:
address_groups = self.plugin._get_address_groups(
context, router_id, network_id)
edge_utils.update_internal_interface(
self.nsx_v, context, router_id, network_id,
address_groups, router_db.admin_state_up)
with locking.LockManager.get_lock(edge_id):
for network_id in intf_net_ids:
address_groups = self.plugin._get_address_groups(
context, router_id, network_id)
edge_utils.update_internal_interface(
self.nsx_v, context, router_id, network_id,
address_groups, router_db.admin_state_up)
# Update external interface (which also update nat rules, routes, etc)
external_net_id = self._get_external_network_id_by_router(context,

10
vmware_nsx/plugins/nsx_v/md_proxy.py
View File

@ -453,7 +453,8 @@ class NsxVMetadataProxyHandler(object):
'name': None,
'admin_state_up': True,
'device_id': rtr_id,
'device_owner': constants.DEVICE_OWNER_ROUTER_INTF,
'device_owner': (constants.DEVICE_OWNER_NETWORK_PREFIX +
'md_interface'),
'fixed_ips': constants.ATTR_NOT_SPECIFIED,
'mac_address': constants.ATTR_NOT_SPECIFIED,
'port_security_enabled': False,
@ -465,9 +466,10 @@ class NsxVMetadataProxyHandler(object):
context, self.internal_net, rtr_id, is_proxy=True)
edge_ip = port['fixed_ips'][0]['ip_address']
edge_utils.update_internal_interface(
self.nsxv_plugin.nsx_v, context, rtr_id,
self.internal_net, address_groups)
with locking.LockManager.get_lock(edge_id):
edge_utils.update_internal_interface(
self.nsxv_plugin.nsx_v, context, rtr_id,
self.internal_net, address_groups)
self._setup_metadata_lb(rtr_id,
port['fixed_ips'][0]['ip_address'],

27
vmware_nsx/plugins/nsx_v/plugin.py
View File

@ -2619,17 +2619,14 @@ class NsxVPluginV2(addr_pair_db.AllowedAddressPairsMixin,
self.create_port(context, {'port': port_dict})
try:
resource_id = self.edge_manager.create_dhcp_edge_service(
context, network_id, subnet)
self.edge_manager.create_dhcp_edge_service(context, network_id,
subnet)
# Create all dhcp ports within the network
address_groups = self._create_network_dhcp_address_group(
context, network_id)
self.edge_manager.update_dhcp_edge_service(
context, network_id, address_groups=address_groups)
if resource_id:
self._update_dhcp_service_new_edge(context, resource_id)
except Exception:
with excutils.save_and_reraise_exception():
LOG.exception(_LE("Failed to update DHCP for subnet %s"),
@ -3298,15 +3295,17 @@ class NsxVPluginV2(addr_pair_db.AllowedAddressPairsMixin,
# for each one to reflect the router admin-state-up status.
intf_net_ids = (
self._get_internal_network_ids_by_router(context, router_id))
for network_id in intf_net_ids:
address_groups = (
self._get_address_groups(context, router_id, network_id))
update_args = (self.nsx_v, context, router_id, network_id,
address_groups, admin_state)
if router_type == 'distributed':
edge_utils.update_vdr_internal_interface(*update_args)
else:
edge_utils.update_internal_interface(*update_args)
edge_id = self._get_edge_id_by_rtr_id(context, router_id)
with locking.LockManager.get_lock(edge_id):
for network_id in intf_net_ids:
address_groups = (
self._get_address_groups(context, router_id, network_id))
update_args = (self.nsx_v, context, router_id, network_id,
address_groups, admin_state)
if router_type == 'distributed':
edge_utils.update_vdr_internal_interface(*update_args)
else:
edge_utils.update_internal_interface(*update_args)
def _get_interface_info_net_id(self, context, interface_info):
is_port, is_sub = self._validate_interface_info(interface_info)

4
vmware_nsx/plugins/nsx_v/vshield/edge_appliance_driver.py
View File

@ -382,7 +382,7 @@ class EdgeApplianceDriver(object):
def deploy_edge(self, context, router_id, name, internal_network,
dist=False, loadbalancer_enable=True,
appliance_size=nsxv_constants.LARGE,
availability_zone=None):
availability_zone=None, deploy_metadata=False):
edge_name = name
edge = self._assemble_edge(
@ -444,7 +444,7 @@ class EdgeApplianceDriver(object):
self.callbacks.complete_edge_creation(
context, edge_id, name, router_id, dist, True,
availability_zone)
availability_zone, deploy_metadata)
except exceptions.VcnsApiException:
self.callbacks.complete_edge_creation(

85
vmware_nsx/plugins/nsx_v/vshield/edge_utils.py
View File

@ -248,7 +248,7 @@ class EdgeManager(object):
def _deploy_edge(self, context, lrouter,
lswitch=None, appliance_size=nsxv_constants.COMPACT,
edge_type=nsxv_constants.SERVICE_EDGE,
availability_zone=None):
availability_zone=None, deploy_metadata=False):
"""Create an edge for logical router support."""
if context is None:
context = q_context.get_admin_context()
@ -257,7 +257,8 @@ class EdgeManager(object):
lrouter['name'], internal_network=None,
appliance_size=appliance_size,
dist=(edge_type == nsxv_constants.VDR_EDGE),
availability_zone=availability_zone)
availability_zone=availability_zone,
deploy_metadata=deploy_metadata)
def _deploy_backup_edges_on_db(self, context, num,
appliance_size=nsxv_constants.COMPACT,
@ -650,7 +651,8 @@ class EdgeManager(object):
def _allocate_edge_appliance(self, context, resource_id, name,
appliance_size=nsxv_constants.COMPACT,
dist=False,
availability_zone=None):
availability_zone=None,
deploy_metadata=False):
"""Try to allocate one available edge from pool."""
edge_type = (nsxv_constants.VDR_EDGE if dist else
nsxv_constants.SERVICE_EDGE)
@ -694,7 +696,8 @@ class EdgeManager(object):
edge_id = self._deploy_edge(context, lrouter,
appliance_size=appliance_size,
edge_type=edge_type,
availability_zone=availability_zone)
availability_zone=availability_zone,
deploy_metadata=deploy_metadata)
else:
LOG.debug("Select edge: %(edge_id)s from pool for %(name)s",
{'edge_id': available_router_binding['edge_id'],
@ -712,16 +715,20 @@ class EdgeManager(object):
edge_type=edge_type,
availability_zone=availability_zone.name)
edge_id = available_router_binding['edge_id']
LOG.debug("Select edge: %(edge_id)s from pool for %(name)s",
{'edge_id': edge_id, 'name': name})
with locking.LockManager.get_lock(str(edge_id)):
self.nsxv_manager.callbacks.complete_edge_creation(
context, edge_id, lrouter['name'], lrouter['id'], dist,
True)
True, deploy_metadata)
# change edge's name at backend
self.nsxv_manager.update_edge(
context, resource_id, available_router_binding['edge_id'],
name, None, appliance_size=appliance_size, dist=dist,
set_errors=True, availability_zone=availability_zone)
try:
self.nsxv_manager.rename_edge(edge_id, name)
except nsxapi_exc.VcnsApiException as e:
LOG.error(_LE("Failed to update edge: %s"),
e.response)
self.nsxv_manager.callbacks.complete_edge_update(
context, edge_id, resource_id, False, set_errors=True)
backup_num = len(self._get_backup_edge_bindings(
context, appliance_size=appliance_size, edge_type=edge_type,
@ -824,7 +831,8 @@ class EdgeManager(object):
self._allocate_edge_appliance(
context, resource_id, resource_name,
appliance_size=vcns_const.SERVICE_SIZE_MAPPING['dhcp'],
availability_zone=availability_zone)
availability_zone=availability_zone,
deploy_metadata=True)
def allocate_lb_edge_appliance(
self, context, resource_id, availability_zone,
@ -2020,14 +2028,7 @@ class EdgeManager(object):
def update_external_interface(
self, nsxv_manager, context, router_id, ext_net_id,
ipaddr, netmask, secondary=None):
with locking.LockManager.get_lock(str(router_id)):
self._update_external_interface(nsxv_manager, context, router_id,
ext_net_id, ipaddr, netmask,
secondary=secondary)
def _update_external_interface(
self, nsxv_manager, context, router_id, ext_net_id,
ipaddr, netmask, secondary=None):
secondary = secondary or []
binding = nsxv_db.get_nsxv_router_binding(context.session, router_id)
@ -2326,14 +2327,6 @@ def _check_ipnet_ip(ipnet, ip_address):
def update_internal_interface(nsxv_manager, context, router_id, int_net_id,
address_groups, is_connected=True):
with locking.LockManager.get_lock(str(router_id)):
_update_internal_interface(nsxv_manager, context, router_id,
int_net_id, address_groups,
is_connected=is_connected)
def _update_internal_interface(nsxv_manager, context, router_id, int_net_id,
address_groups, is_connected=True):
# Get edge id
binding = nsxv_db.get_nsxv_router_binding(context.session, router_id)
@ -2363,14 +2356,6 @@ def _update_internal_interface(nsxv_manager, context, router_id, int_net_id,
def add_vdr_internal_interface(nsxv_manager, context, router_id,
int_net_id, address_groups, is_connected=True):
with locking.LockManager.get_lock(str(router_id)):
_add_vdr_internal_interface(nsxv_manager, context, router_id,
int_net_id, address_groups,
is_connected=is_connected)
def _add_vdr_internal_interface(nsxv_manager, context, router_id,
int_net_id, address_groups, is_connected=True):
# Get edge id
binding = nsxv_db.get_nsxv_router_binding(context.session, router_id)
edge_id = binding['edge_id']
@ -2398,15 +2383,6 @@ def _add_vdr_internal_interface(nsxv_manager, context, router_id,
def update_vdr_internal_interface(nsxv_manager, context, router_id, int_net_id,
address_groups, is_connected=True):
with locking.LockManager.get_lock(str(router_id)):
_update_vdr_internal_interface(nsxv_manager, context, router_id,
int_net_id, address_groups,
is_connected=is_connected)
def _update_vdr_internal_interface(nsxv_manager, context, router_id,
int_net_id, address_groups,
is_connected=True):
# Get edge id
binding = nsxv_db.get_nsxv_router_binding(context.session, router_id)
edge_id = binding['edge_id']
@ -2426,13 +2402,7 @@ def _update_vdr_internal_interface(nsxv_manager, context, router_id,
def delete_interface(nsxv_manager, context, router_id, network_id, dist=False):
with locking.LockManager.get_lock(str(router_id)):
_delete_interface(nsxv_manager, context, router_id, network_id,
dist=dist)
def _delete_interface(nsxv_manager, context, router_id, network_id,
dist=False):
# Get edge id
binding = nsxv_db.get_nsxv_router_binding(context.session, router_id)
if not binding:
@ -2640,9 +2610,9 @@ class NsxVCallbacks(object):
else:
self._vcm = None
def complete_edge_creation(
self, context, edge_id, name, router_id, dist, deploy_successful,
availability_zone=None):
def complete_edge_creation(self, context, edge_id, name, router_id, dist,
deploy_successful, availability_zone=None,
deploy_metadata=False):
router_db = None
if uuidutils.is_uuid_like(router_id):
try:
@ -2652,6 +2622,17 @@ class NsxVCallbacks(object):
LOG.warning(_LW("Router %s not found"), name)
if deploy_successful:
metadata_proxy_handler = self.plugin.get_metadata_proxy_handler(
availability_zone)
if deploy_metadata and metadata_proxy_handler:
LOG.debug('Update metadata for resource %s',
router_id)
metadata_proxy_handler.configure_router_edge(
context, router_id)
self.plugin.setup_dhcp_edge_fw_rules(context, self.plugin,
router_id)
LOG.debug("Successfully deployed %(edge_id)s for router %(name)s",
{'edge_id': edge_id,
'name': name})

2
vmware_nsx/tests/unit/nsx_v/test_md_proxy.py
View File

@ -47,7 +47,7 @@ class NsxVPluginWithMdV2TestCase(test_plugin.NsxVPluginV2TestCase):
mock_alloc_vnic = mock.patch.object(nsxv_db, 'allocate_edge_vnic')
mock_alloc_vnic_inst = mock_alloc_vnic.start()
mock_alloc_vnic_inst.return_value = nsxv_models.NsxvEdgeVnicBinding
mock.patch.object(edge_utils, "_update_internal_interface").start()
mock.patch.object(edge_utils, "update_internal_interface").start()
super(NsxVPluginWithMdV2TestCase, self).setUp(
plugin=plugin, ext_mgr=ext_mgr,

6
vmware_nsx/tests/unit/nsx_v/test_plugin.py
View File

@ -3255,7 +3255,7 @@ class TestExclusiveRouterTestCase(L3NatTest, L3NatTestCaseBase,
TestExclusiveRouterTestCase,
self).test_router_add_interface_multiple_ipv6_subnets_same_net()
def _fake_update_edge(self, edge_id, request):
def _fake_rename_edge(self, edge_id, name):
raise vcns_exc.VcnsApiException(
status=400, header={'status': 200}, uri='fake_url', response='')
@ -3272,8 +3272,8 @@ class TestExclusiveRouterTestCase(L3NatTest, L3NatTestCaseBase,
return_value=available_edge):
# Mock for update_edge task failure
with mock.patch.object(
p.nsx_v.vcns, 'update_edge',
side_effect=self._fake_update_edge):
p.edge_manager.nsxv_manager, 'rename_edge',
side_effect=self._fake_rename_edge):
router = {'router': {'admin_state_up': True,
'name': 'e161be1d-0d0d-4046-9823-5a593d94f72c',
'tenant_id': context.get_admin_context().tenant_id,

26
vmware_nsx/tests/unit/nsx_v/vshield/test_edge_utils.py
View File

@ -128,12 +128,8 @@ class EdgeDHCPManagerTestCase(EdgeUtilsTestCaseMixin):
self.edge_manager.create_dhcp_edge_service(self.ctx,
fake_network['id'],
fake_subnet)
resource_id = (vcns_const.DHCP_EDGE_PREFIX + fake_network['id'])[:36]
self.nsxv_manager.update_edge.assert_called_once_with(
self.ctx, resource_id, 'edge-1', mock.ANY, None,
appliance_size=vcns_const.SERVICE_SIZE_MAPPING['dhcp'],
dist=False, set_errors=True,
availability_zone=mock.ANY)
self.nsxv_manager.rename_edge.assert_called_once_with('edge-1',
mock.ANY)
def test_get_random_available_edge(self):
available_edge_ids = ['edge-1', 'edge-2']
@ -700,10 +696,8 @@ class EdgeManagerTestCase(EdgeUtilsTestCaseMixin):
availability_zone=self.az)
edge_id = (EDGE_AVAIL + nsxv_constants.LARGE + '-' +
nsxv_constants.SERVICE_EDGE + '-edge-' + str(0))
self.nsxv_manager.update_edge.assert_has_calls(
[mock.call(self.ctx, 'fake_id', edge_id, 'fake_name', None,
set_errors=True, appliance_size=nsxv_constants.LARGE,
dist=False, availability_zone=self.az)])
self.nsxv_manager.rename_edge.assert_has_calls(
[mock.call(edge_id, 'fake_name')])
def test_allocate_compact_edge_appliance_with_default(self):
self.edge_manager.edge_pool_dicts = self.default_edge_pool_dicts
@ -719,10 +713,8 @@ class EdgeManagerTestCase(EdgeUtilsTestCaseMixin):
availability_zone=self.az)
edge_id = (EDGE_AVAIL + nsxv_constants.COMPACT + '-' +
nsxv_constants.SERVICE_EDGE + '-edge-' + str(0))
self.nsxv_manager.update_edge.assert_has_calls(
[mock.call(self.ctx, 'fake_id', edge_id, 'fake_name', None,
set_errors=True, appliance_size=nsxv_constants.COMPACT,
dist=False, availability_zone=self.az)])
self.nsxv_manager.rename_edge.assert_has_calls(
[mock.call(edge_id, 'fake_name')])
def test_allocate_large_edge_appliance_with_vdr(self):
self.edge_manager.edge_pool_dicts = self.vdr_edge_pool_dicts
@ -738,10 +730,8 @@ class EdgeManagerTestCase(EdgeUtilsTestCaseMixin):
availability_zone=self.az)
edge_id = (EDGE_AVAIL + nsxv_constants.LARGE + '-' +
nsxv_constants.VDR_EDGE + '-edge-' + str(0))
self.nsxv_manager.update_edge.assert_has_calls(
[mock.call(self.ctx, 'fake_id', edge_id, 'fake_name', None,
set_errors=True, appliance_size=nsxv_constants.LARGE,
dist=True, availability_zone=self.az)])
self.nsxv_manager.rename_edge.assert_has_calls(
[mock.call(edge_id, 'fake_name')])
def test_free_edge_appliance_with_empty(self):
self.edge_manager._clean_all_error_edge_bindings = mock.Mock()

2
vmware_nsx/tests/unit/nsx_v/vshield/test_vcns_driver.py
View File

@ -347,7 +347,7 @@ class VcnsDriverTestCase(base.BaseTestCase):
def complete_edge_creation(
self, context, edge_id, name, router_id, dist, deploy_successful,
availability_zone=None):
availability_zone=None, deploy_metadata=False):
pass
def _deploy_edge(self):