summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGary Kotton <gkotton@vmware.com>2017-05-06 03:12:22 +0300
committerGary Kotton <gkotton@vmware.com>2017-06-07 01:55:02 -0700
commit3eb98e5070cd8743a549cbbfb9c363849b87d09a (patch)
tree086226021508685cd950b1582740c28fd56346ff
parent337d3f0dd680c6d47e37bc83a853d5ff69e356fe (diff)
NSX|V: prevent deadlock with subnet creation and deletion
Ensure that we treat the locks in the same order with the delete and create operations. The fix above result in a deadlcok due to a change in neutron where ipam delete call port_update. This conflicted with commit d89eba1a851d76179bd882cc8024b86f71e1153b. To ensure locking for this we make use of a new lock for the DHCP interfaces. Change-Id: I6c3f25ab40247853024560c00d3faa106e5d90b8
Notes
Notes (review): Verified+1: VMware NSX CI Code-Review+2: garyk <gkotton@vmware.com> Workflow+1: garyk <gkotton@vmware.com> Code-Review+2: Tong Liu <tongl@vmware.com> Verified+2: Jenkins Submitted-by: Jenkins Submitted-at: Thu, 08 Jun 2017 06:21:49 +0000 Reviewed-on: https://review.openstack.org/471641 Project: openstack/vmware-nsx Branch: refs/heads/stable/ocata
-rw-r--r--vmware_nsx/plugins/nsx_v/drivers/distributed_router_driver.py6
-rw-r--r--vmware_nsx/plugins/nsx_v/plugin.py67
-rw-r--r--vmware_nsx/plugins/nsx_v/vshield/edge_utils.py6
3 files changed, 36 insertions, 43 deletions
diff --git a/vmware_nsx/plugins/nsx_v/drivers/distributed_router_driver.py b/vmware_nsx/plugins/nsx_v/drivers/distributed_router_driver.py
index 4c96f0e..0739bf5 100644
--- a/vmware_nsx/plugins/nsx_v/drivers/distributed_router_driver.py
+++ b/vmware_nsx/plugins/nsx_v/drivers/distributed_router_driver.py
@@ -485,11 +485,7 @@ class RouterDistributedDriver(router_driver.RouterBaseDriver):
485 with locking.LockManager.get_lock(network_id): 485 with locking.LockManager.get_lock(network_id):
486 dhcp_id = self.edge_manager.create_dhcp_edge_service( 486 dhcp_id = self.edge_manager.create_dhcp_edge_service(
487 context, network_id, subnet) 487 context, network_id, subnet)
488 488 self.plugin._update_dhcp_adddress(context, network_id)
489 address_groups = self.plugin._create_network_dhcp_address_group(
490 context, network_id)
491 self.edge_manager.update_dhcp_edge_service(
492 context, network_id, address_groups=address_groups)
493 if dhcp_id: 489 if dhcp_id:
494 edge_id, az_name = self.plugin._get_edge_id_and_az_by_rtr_id( 490 edge_id, az_name = self.plugin._get_edge_id_and_az_by_rtr_id(
495 context, dhcp_id) 491 context, dhcp_id)
diff --git a/vmware_nsx/plugins/nsx_v/plugin.py b/vmware_nsx/plugins/nsx_v/plugin.py
index 5a3a03d..b4c46b6 100644
--- a/vmware_nsx/plugins/nsx_v/plugin.py
+++ b/vmware_nsx/plugins/nsx_v/plugin.py
@@ -1768,6 +1768,13 @@ class NsxVPluginV2(addr_pair_db.AllowedAddressPairsMixin,
1768 return self._update_port(context, id, port, original_port, 1768 return self._update_port(context, id, port, original_port,
1769 is_compute_port, device_id) 1769 is_compute_port, device_id)
1770 1770
1771 def _update_dhcp_adddress(self, context, network_id):
1772 with locking.LockManager.get_lock('dhcp-update-%s' % network_id):
1773 address_groups = self._create_network_dhcp_address_group(
1774 context, network_id)
1775 self._update_dhcp_edge_service(context, network_id,
1776 address_groups)
1777
1771 def _update_port(self, context, id, port, original_port, is_compute_port, 1778 def _update_port(self, context, id, port, original_port, is_compute_port,
1772 device_id): 1779 device_id):
1773 attrs = port[attr.PORT] 1780 attrs = port[attr.PORT]
@@ -1896,10 +1903,8 @@ class NsxVPluginV2(addr_pair_db.AllowedAddressPairsMixin,
1896 self._create_dhcp_static_binding(context, ret_port) 1903 self._create_dhcp_static_binding(context, ret_port)
1897 elif owner == constants.DEVICE_OWNER_DHCP: 1904 elif owner == constants.DEVICE_OWNER_DHCP:
1898 # Update the ip of the dhcp port 1905 # Update the ip of the dhcp port
1899 address_groups = self._create_network_dhcp_address_group( 1906 self._update_dhcp_adddress(context,
1900 context, ret_port['network_id']) 1907 ret_port['network_id'])
1901 self._update_dhcp_edge_service(
1902 context, ret_port['network_id'], address_groups)
1903 elif (owner == constants.DEVICE_OWNER_ROUTER_GW or 1908 elif (owner == constants.DEVICE_OWNER_ROUTER_GW or
1904 owner == constants.DEVICE_OWNER_ROUTER_INTF): 1909 owner == constants.DEVICE_OWNER_ROUTER_INTF):
1905 # This is a router port - update the edge appliance 1910 # This is a router port - update the edge appliance
@@ -2116,33 +2121,33 @@ class NsxVPluginV2(addr_pair_db.AllowedAddressPairsMixin,
2116 # and send update dhcp interface rest call before deleting subnet's 2121 # and send update dhcp interface rest call before deleting subnet's
2117 # corresponding dhcp interface rest call and lead to overlap response 2122 # corresponding dhcp interface rest call and lead to overlap response
2118 # from backend. 2123 # from backend.
2119 with locking.LockManager.get_lock('nsx-edge-pool'): 2124 network_id = subnet['network_id']
2120 with context.session.begin(subtransactions=True): 2125 with locking.LockManager.get_lock(network_id):
2121 super(NsxVPluginV2, self).delete_subnet(context, id) 2126 with locking.LockManager.get_lock('nsx-edge-pool'):
2127 with context.session.begin(subtransactions=True):
2128 super(NsxVPluginV2, self).delete_subnet(context, id)
2129
2122 if subnet['enable_dhcp']: 2130 if subnet['enable_dhcp']:
2123 # There is only DHCP port available 2131 # There is only DHCP port available
2124 if len(ports) == 1: 2132 if len(ports) == 1:
2125 port = ports.pop() 2133 port = ports.pop()
2134 # This is done out of the transaction as it invokes
2135 # update_port which interfaces with the NSX
2126 self.ipam.delete_port(context, port['id']) 2136 self.ipam.delete_port(context, port['id'])
2127 2137
2128 if subnet['enable_dhcp']: 2138 # Delete the DHCP edge service
2129 # Delete the DHCP edge service 2139 filters = {'network_id': [network_id]}
2130 network_id = subnet['network_id'] 2140 remaining_subnets = self.get_subnets(context,
2131 filters = {'network_id': [network_id]} 2141 filters=filters)
2132 remaining_subnets = self.get_subnets(context, 2142 if len(remaining_subnets) == 0:
2133 filters=filters) 2143 self._cleanup_dhcp_edge_before_deletion(
2134 if len(remaining_subnets) == 0: 2144 context, network_id)
2135 self._cleanup_dhcp_edge_before_deletion( 2145 LOG.debug("Delete the DHCP service for network %s",
2136 context, network_id) 2146 network_id)
2137 LOG.debug("Delete the DHCP service for network %s", 2147 self._delete_dhcp_edge_service(context, network_id)
2138 network_id) 2148 else:
2139 self._delete_dhcp_edge_service(context, network_id) 2149 # Update address group and delete the DHCP port only
2140 else: 2150 self._update_dhcp_adddress(context, network_id)
2141 # Update address group and delete the DHCP port only
2142 address_groups = self._create_network_dhcp_address_group(
2143 context, network_id)
2144 self._update_dhcp_edge_service(context, network_id,
2145 address_groups)
2146 2151
2147 def _is_overlapping_reserved_subnets(self, subnet): 2152 def _is_overlapping_reserved_subnets(self, subnet):
2148 """Return True if the subnet overlaps with reserved subnets. 2153 """Return True if the subnet overlaps with reserved subnets.
@@ -2265,7 +2270,8 @@ class NsxVPluginV2(addr_pair_db.AllowedAddressPairsMixin,
2265 self._update_dhcp_service_with_subnet(context, s) 2270 self._update_dhcp_service_with_subnet(context, s)
2266 except Exception: 2271 except Exception:
2267 with excutils.save_and_reraise_exception(): 2272 with excutils.save_and_reraise_exception():
2268 self.delete_subnet(context, s['id']) 2273 super(NsxVPluginV2, self).delete_subnet(context,
2274 s['id'])
2269 return s 2275 return s
2270 2276
2271 def _process_subnet_ext_attr_create(self, session, subnet_db, 2277 def _process_subnet_ext_attr_create(self, session, subnet_db,
@@ -2467,9 +2473,7 @@ class NsxVPluginV2(addr_pair_db.AllowedAddressPairsMixin,
2467 network_id) 2473 network_id)
2468 self._delete_dhcp_edge_service(context, network_id) 2474 self._delete_dhcp_edge_service(context, network_id)
2469 return 2475 return
2470 address_groups = self._create_network_dhcp_address_group(context, 2476 self._update_dhcp_adddress(context, network_id)
2471 network_id)
2472 self._update_dhcp_edge_service(context, network_id, address_groups)
2473 2477
2474 def _get_conflict_network_ids_by_overlapping(self, context, subnets): 2478 def _get_conflict_network_ids_by_overlapping(self, context, subnets):
2475 with locking.LockManager.get_lock('nsx-networking'): 2479 with locking.LockManager.get_lock('nsx-networking'):
@@ -2567,10 +2571,7 @@ class NsxVPluginV2(addr_pair_db.AllowedAddressPairsMixin,
2567 self.edge_manager.create_dhcp_edge_service(context, network_id, 2571 self.edge_manager.create_dhcp_edge_service(context, network_id,
2568 subnet) 2572 subnet)
2569 # Create all dhcp ports within the network 2573 # Create all dhcp ports within the network
2570 address_groups = self._create_network_dhcp_address_group( 2574 self._update_dhcp_adddress(context, network_id)
2571 context, network_id)
2572 self.edge_manager.update_dhcp_edge_service(
2573 context, network_id, address_groups=address_groups)
2574 2575
2575 except Exception: 2576 except Exception:
2576 with excutils.save_and_reraise_exception(): 2577 with excutils.save_and_reraise_exception():
diff --git a/vmware_nsx/plugins/nsx_v/vshield/edge_utils.py b/vmware_nsx/plugins/nsx_v/vshield/edge_utils.py
index f718d82..271f8da 100644
--- a/vmware_nsx/plugins/nsx_v/vshield/edge_utils.py
+++ b/vmware_nsx/plugins/nsx_v/vshield/edge_utils.py
@@ -1492,11 +1492,7 @@ class EdgeManager(object):
1492 LOG.error(_LE('Database conflict could not be recovered ' 1492 LOG.error(_LE('Database conflict could not be recovered '
1493 'for VDR %(vdr)s DHCP edge %(dhcp)s'), 1493 'for VDR %(vdr)s DHCP edge %(dhcp)s'),
1494 {'vdr': vdr_router_id, 'dhcp': dhcp_edge_id}) 1494 {'vdr': vdr_router_id, 'dhcp': dhcp_edge_id})
1495 1495 self.plugin._update_dhcp_adddress(context, network_id)
1496 address_groups = self.plugin._create_network_dhcp_address_group(
1497 context, network_id)
1498 self.update_dhcp_edge_service(
1499 context, network_id, address_groups=address_groups)
1500 1496
1501 self.set_sysctl_rp_filter_for_vdr_dhcp( 1497 self.set_sysctl_rp_filter_for_vdr_dhcp(
1502 context, dhcp_edge_id, network_id) 1498 context, dhcp_edge_id, network_id)