diff --git a/etc/neutron.conf b/etc/neutron.conf index 2ba7ada42..083662642 100644 --- a/etc/neutron.conf +++ b/etc/neutron.conf @@ -172,6 +172,22 @@ lock_path = $state_path/lock # =========== end of items for agent scheduler extension ===== +# =========== items for l3 extension ============== +# Enable high availability for virtual routers. +# l3_ha = False +# +# Maximum number of l3 agents which a HA router will be scheduled on. If it +# is set to 0 the router will be scheduled on every agent. +# max_l3_agents_per_router = 3 +# +# Minimum number of l3 agents which a HA router will be scheduled on. The +# default value is 2. +# min_l3_agents_per_router = 2 +# +# CIDR of the administrative network if HA mode is enabled +# l3_ha_net_cidr = 169.254.192.0/18 +# =========== end of items for l3 extension ======= + # =========== WSGI parameters related to the API server ============== # Number of separate worker processes to spawn. The default, 0, runs the # worker thread in the current process. Greater than 0 launches that number of diff --git a/etc/policy.json b/etc/policy.json index c5aec3b3e..e7db43575 100644 --- a/etc/policy.json +++ b/etc/policy.json @@ -58,13 +58,16 @@ "update_port:mac_learning_enabled": "rule:admin_or_network_owner", "delete_port": "rule:admin_or_owner", + "get_router:ha": "rule:admin_only", "create_router": "rule:regular_user", "create_router:external_gateway_info:enable_snat": "rule:admin_only", "create_router:distributed": "rule:admin_only", + "create_router:ha": "rule:admin_only", "get_router": "rule:admin_or_owner", "get_router:distributed": "rule:admin_only", "update_router:external_gateway_info:enable_snat": "rule:admin_only", "update_router:distributed": "rule:admin_only", + "update_router:ha": "rule:admin_only", "delete_router": "rule:admin_or_owner", "add_router_interface": "rule:admin_or_owner", diff --git a/neutron/api/rpc/handlers/l3_rpc.py b/neutron/api/rpc/handlers/l3_rpc.py index f31209dc9..f5c7389d5 100644 --- a/neutron/api/rpc/handlers/l3_rpc.py +++ b/neutron/api/rpc/handlers/l3_rpc.py @@ -38,7 +38,8 @@ class L3RpcCallback(n_rpc.RpcCallback): # 1.1 Support update_floatingip_statuses # 1.2 Added methods for DVR support # 1.3 Added a method that returns the list of activated services - RPC_API_VERSION = '1.3' + # 1.4 Added L3 HA update_router_state + RPC_API_VERSION = '1.4' @property def plugin(self): @@ -104,6 +105,10 @@ class L3RpcCallback(n_rpc.RpcCallback): for interface in router.get(constants.INTERFACE_KEY, []): self._ensure_host_set_on_port(context, host, interface, router['id']) + interface = router.get(constants.HA_INTERFACE_KEY) + if interface: + self._ensure_host_set_on_port(context, host, interface, + router['id']) def _ensure_host_set_on_port(self, context, host, port, router_id=None): if (port and @@ -224,3 +229,11 @@ class L3RpcCallback(n_rpc.RpcCallback): 'and on host %(host)s', {'snat_port_list': snat_port_list, 'host': host}) return snat_port_list + + def update_router_state(self, context, **kwargs): + router_id = kwargs.get('router_id') + state = kwargs.get('state') + host = kwargs.get('host') + + return self.l3plugin.update_router_state(context, router_id, state, + host=host) diff --git a/neutron/common/constants.py b/neutron/common/constants.py index f1c15c535..b053b1ae3 100644 --- a/neutron/common/constants.py +++ b/neutron/common/constants.py @@ -29,6 +29,7 @@ FLOATINGIP_STATUS_ACTIVE = 'ACTIVE' FLOATINGIP_STATUS_DOWN = 'DOWN' FLOATINGIP_STATUS_ERROR = 'ERROR' +DEVICE_OWNER_ROUTER_HA_INTF = "network:router_ha_interface" DEVICE_OWNER_ROUTER_INTF = "network:router_interface" DEVICE_OWNER_ROUTER_GW = "network:router_gateway" DEVICE_OWNER_FLOATINGIP = "network:floatingip" @@ -42,10 +43,17 @@ DEVICE_ID_RESERVED_DHCP_PORT = "reserved_dhcp_port" FLOATINGIP_KEY = '_floatingips' INTERFACE_KEY = '_interfaces' +HA_INTERFACE_KEY = '_ha_interface' +HA_ROUTER_STATE_KEY = '_ha_state' METERING_LABEL_KEY = '_metering_labels' FLOATINGIP_AGENT_INTF_KEY = '_floatingip_agent_interfaces' SNAT_ROUTER_INTF_KEY = '_snat_router_interfaces' +HA_NETWORK_NAME = 'HA network tenant %s' +HA_SUBNET_NAME = 'HA subnet tenant %s' +HA_PORT_NAME = 'HA port tenant %s' +MINIMUM_AGENTS_FOR_HA = 2 + IPv4 = 'IPv4' IPv6 = 'IPv6' @@ -101,6 +109,7 @@ L3_AGENT_SCHEDULER_EXT_ALIAS = 'l3_agent_scheduler' DHCP_AGENT_SCHEDULER_EXT_ALIAS = 'dhcp_agent_scheduler' LBAAS_AGENT_SCHEDULER_EXT_ALIAS = 'lbaas_agent_scheduler' L3_DISTRIBUTED_EXT_ALIAS = 'dvr' +L3_HA_MODE_EXT_ALIAS = 'l3-ha' # Protocol names and numbers for Security Groups/Firewalls PROTO_NAME_TCP = 'tcp' diff --git a/neutron/db/l3_agentschedulers_db.py b/neutron/db/l3_agentschedulers_db.py index e9dc8e308..2d52921fd 100644 --- a/neutron/db/l3_agentschedulers_db.py +++ b/neutron/db/l3_agentschedulers_db.py @@ -284,8 +284,14 @@ class L3AgentSchedulerDbMixin(l3agentscheduler.L3AgentSchedulerPluginBase, RouterL3AgentBinding.router_id.in_(router_ids)) router_ids = [item[0] for item in query] if router_ids: - return self.get_sync_data(context, router_ids=router_ids, - active=True) + if n_utils.is_extension_supported(self, + constants.L3_HA_MODE_EXT_ALIAS): + return self.get_ha_sync_data_for_host(context, host, + router_ids=router_ids, + active=True) + else: + return self.get_sync_data(context, router_ids=router_ids, + active=True) else: return [] diff --git a/neutron/db/l3_attrs_db.py b/neutron/db/l3_attrs_db.py index d43cdc7b4..7c82f84af 100644 --- a/neutron/db/l3_attrs_db.py +++ b/neutron/db/l3_attrs_db.py @@ -40,6 +40,11 @@ class RouterExtraAttributes(model_base.BASEV2): service_router = sa.Column(sa.Boolean, default=False, server_default=sa.sql.false(), nullable=False) + ha = sa.Column(sa.Boolean, default=False, + server_default=sa.sql.false(), + nullable=False) + ha_vr_id = sa.Column(sa.Integer()) + router = orm.relationship( l3_db.Router, backref=orm.backref("extra_attributes", lazy='joined', diff --git a/neutron/db/l3_dvr_db.py b/neutron/db/l3_dvr_db.py index 6a91c0fe0..695617192 100644 --- a/neutron/db/l3_dvr_db.py +++ b/neutron/db/l3_dvr_db.py @@ -61,7 +61,7 @@ class L3_NAT_with_dvr_db_mixin(l3_db.L3_NAT_db_mixin, def _create_router_db(self, context, router, tenant_id): """Create a router db object with dvr additions.""" - router['distributed'] = _is_distributed_router(router) + router['distributed'] = is_distributed_router(router) with context.session.begin(subtransactions=True): router_db = super( L3_NAT_with_dvr_db_mixin, self)._create_router_db( @@ -128,7 +128,7 @@ class L3_NAT_with_dvr_db_mixin(l3_db.L3_NAT_db_mixin, router_is_uuid = isinstance(router, basestring) if router_is_uuid: router = self._get_router(context, router) - if _is_distributed_router(router): + if is_distributed_router(router): return DEVICE_OWNER_DVR_INTERFACE return super(L3_NAT_with_dvr_db_mixin, self)._get_device_owner(context, router) @@ -534,7 +534,7 @@ class L3_NAT_with_dvr_db_mixin(l3_db.L3_NAT_db_mixin, l3_port_check=False) -def _is_distributed_router(router): +def is_distributed_router(router): """Return True if router to be handled is distributed.""" try: # See if router is a DB object first diff --git a/neutron/db/l3_hamode_db.py b/neutron/db/l3_hamode_db.py new file mode 100644 index 000000000..19ecf3cc9 --- /dev/null +++ b/neutron/db/l3_hamode_db.py @@ -0,0 +1,459 @@ +# Copyright (C) 2014 eNovance SAS +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +import netaddr +from oslo.config import cfg +from oslo.db import exception as db_exc +import sqlalchemy as sa +from sqlalchemy import orm + +from neutron.api.v2 import attributes +from neutron.common import constants +from neutron.db import agents_db +from neutron.db import l3_dvr_db +from neutron.db import model_base +from neutron.db import models_v2 +from neutron.extensions import l3_ext_ha_mode as l3_ha +from neutron.openstack.common import excutils +from neutron.openstack.common.gettextutils import _LI +from neutron.openstack.common.gettextutils import _LW +from neutron.openstack.common import log as logging + +VR_ID_RANGE = set(range(1, 255)) +MAX_ALLOCATION_TRIES = 10 + +LOG = logging.getLogger(__name__) + +L3_HA_OPTS = [ + cfg.BoolOpt('l3_ha', + default=False, + help=_('Enable HA mode for virtual routers.')), + cfg.IntOpt('max_l3_agents_per_router', + default=3, + help=_('Maximum number of agents on which a router will be ' + 'scheduled.')), + cfg.IntOpt('min_l3_agents_per_router', + default=constants.MINIMUM_AGENTS_FOR_HA, + help=_('Minimum number of agents on which a router will be ' + 'scheduled.')), + cfg.StrOpt('l3_ha_net_cidr', + default='169.254.192.0/18', + help=_('Subnet used for the l3 HA admin network.')), +] +cfg.CONF.register_opts(L3_HA_OPTS) + + +class L3HARouterAgentPortBinding(model_base.BASEV2): + """Represent agent binding state of a HA router port. + + A HA Router has one HA port per agent on which it is spawned. + This binding table stores which port is used for a HA router by a + L3 agent. + """ + + __tablename__ = 'ha_router_agent_port_bindings' + + port_id = sa.Column(sa.String(36), sa.ForeignKey('ports.id', + ondelete='CASCADE'), + nullable=False, primary_key=True) + port = orm.relationship(models_v2.Port) + + router_id = sa.Column(sa.String(36), sa.ForeignKey('routers.id', + ondelete='CASCADE'), + nullable=False) + + l3_agent_id = sa.Column(sa.String(36), + sa.ForeignKey("agents.id", + ondelete='CASCADE')) + agent = orm.relationship(agents_db.Agent) + + state = sa.Column(sa.Enum('active', 'standby', name='l3_ha_states'), + default='standby', + server_default='standby') + + +class L3HARouterNetwork(model_base.BASEV2): + """Host HA network for a tenant. + + One HA Network is used per tenant, all HA router ports are created + on this network. + """ + + __tablename__ = 'ha_router_networks' + + tenant_id = sa.Column(sa.String(255), primary_key=True, + nullable=False) + network_id = sa.Column(sa.String(36), + sa.ForeignKey('networks.id', ondelete="CASCADE"), + nullable=False, primary_key=True) + network = orm.relationship(models_v2.Network) + + +class L3HARouterVRIdAllocation(model_base.BASEV2): + """VRID allocation per HA network. + + Keep a track of the VRID allocations per HA network. + """ + + __tablename__ = 'ha_router_vrid_allocations' + + network_id = sa.Column(sa.String(36), + sa.ForeignKey('networks.id', ondelete="CASCADE"), + nullable=False, primary_key=True) + vr_id = sa.Column(sa.Integer(), nullable=False, primary_key=True) + + +class L3_HA_NAT_db_mixin(l3_dvr_db.L3_NAT_with_dvr_db_mixin): + """Mixin class to add high availability capability to routers.""" + + extra_attributes = ( + l3_dvr_db.L3_NAT_with_dvr_db_mixin.extra_attributes + [ + {'name': 'ha', 'default': cfg.CONF.l3_ha}, + {'name': 'ha_vr_id', 'default': 0}]) + + def _verify_configuration(self): + self.ha_cidr = cfg.CONF.l3_ha_net_cidr + try: + net = netaddr.IPNetwork(self.ha_cidr) + except netaddr.AddrFormatError: + raise l3_ha.HANetworkCIDRNotValid(cidr=self.ha_cidr) + if ('/' not in self.ha_cidr or net.network != net.ip): + raise l3_ha.HANetworkCIDRNotValid(cidr=self.ha_cidr) + + if cfg.CONF.min_l3_agents_per_router < constants.MINIMUM_AGENTS_FOR_HA: + raise l3_ha.HAMinimumAgentsNumberNotValid() + + def __init__(self): + self._verify_configuration() + super(L3_HA_NAT_db_mixin, self).__init__() + + def get_ha_network(self, context, tenant_id): + return (context.session.query(L3HARouterNetwork). + filter(L3HARouterNetwork.tenant_id == tenant_id). + first()) + + def _get_allocated_vr_id(self, context, network_id): + with context.session.begin(subtransactions=True): + query = (context.session.query(L3HARouterVRIdAllocation). + filter(L3HARouterVRIdAllocation.network_id == network_id)) + + allocated_vr_ids = set(a.vr_id for a in query) - set([0]) + + return allocated_vr_ids + + def _allocate_vr_id(self, context, network_id, router_id): + for count in range(MAX_ALLOCATION_TRIES): + try: + with context.session.begin(subtransactions=True): + allocated_vr_ids = self._get_allocated_vr_id(context, + network_id) + available_vr_ids = VR_ID_RANGE - allocated_vr_ids + + if not available_vr_ids: + raise l3_ha.NoVRIDAvailable(router_id=router_id) + + allocation = L3HARouterVRIdAllocation() + allocation.network_id = network_id + allocation.vr_id = available_vr_ids.pop() + + context.session.add(allocation) + + return allocation.vr_id + + except db_exc.DBDuplicateEntry: + LOG.info(_LI("Attempt %(count)s to allocate a VRID in the " + "network %(network)s for the router %(router)s"), + {'count': count, 'network': network_id, + 'router': router_id}) + + raise l3_ha.MaxVRIDAllocationTriesReached( + network_id=network_id, router_id=router_id, + max_tries=MAX_ALLOCATION_TRIES) + + def _delete_vr_id_allocation(self, context, ha_network, vr_id): + with context.session.begin(subtransactions=True): + context.session.query(L3HARouterVRIdAllocation).filter_by( + network_id=ha_network.network_id, + vr_id=vr_id).delete() + + def _set_vr_id(self, context, router, ha_network): + with context.session.begin(subtransactions=True): + router.extra_attributes.ha_vr_id = self._allocate_vr_id( + context, ha_network.network_id, router.id) + + def _create_ha_subnet(self, context, network_id, tenant_id): + args = {'subnet': + {'network_id': network_id, + 'tenant_id': '', + 'name': constants.HA_SUBNET_NAME % tenant_id, + 'ip_version': 4, + 'cidr': cfg.CONF.l3_ha_net_cidr, + 'enable_dhcp': False, + 'host_routes': attributes.ATTR_NOT_SPECIFIED, + 'dns_nameservers': attributes.ATTR_NOT_SPECIFIED, + 'allocation_pools': attributes.ATTR_NOT_SPECIFIED, + 'gateway_ip': None}} + return self._core_plugin.create_subnet(context, args) + + def _create_ha_network_tenant_binding(self, context, tenant_id, + network_id): + with context.session.begin(subtransactions=True): + ha_network = L3HARouterNetwork(tenant_id=tenant_id, + network_id=network_id) + context.session.add(ha_network) + return ha_network + + def _create_ha_network(self, context, tenant_id): + admin_ctx = context.elevated() + + args = {'network': + {'name': constants.HA_NETWORK_NAME % tenant_id, + 'tenant_id': '', + 'shared': False, + 'admin_state_up': True, + 'status': constants.NET_STATUS_ACTIVE}} + network = self._core_plugin.create_network(context, args) + try: + ha_network = self._create_ha_network_tenant_binding(admin_ctx, + tenant_id, + network['id']) + except Exception: + with excutils.save_and_reraise_exception(): + self._core_plugin.delete_network(admin_ctx, network['id']) + + try: + self._create_ha_subnet(admin_ctx, network['id'], tenant_id) + except Exception: + with excutils.save_and_reraise_exception(): + self._core_plugin.delete_network(admin_ctx, network['id']) + + return ha_network + + def get_number_of_agents_for_scheduling(self, context): + """Return the number of agents on which the router will be scheduled. + + Raises an exception if there are not enough agents available to honor + the min_agents config parameter. If the max_agents parameter is set to + 0 all the agents will be used. + """ + + min_agents = cfg.CONF.min_l3_agents_per_router + num_agents = len(self.get_l3_agents(context)) + max_agents = cfg.CONF.max_l3_agents_per_router + if max_agents: + if max_agents > num_agents: + LOG.info(_LI("Number of available agents lower than " + "max_l3_agents_per_router. L3 agents " + "available: %s"), num_agents) + else: + num_agents = max_agents + + if num_agents < min_agents: + raise l3_ha.HANotEnoughAvailableAgents(min_agents=min_agents, + num_agents=num_agents) + + return num_agents + + def _create_ha_port_binding(self, context, port_id, router_id): + with context.session.begin(subtransactions=True): + portbinding = L3HARouterAgentPortBinding(port_id=port_id, + router_id=router_id) + context.session.add(portbinding) + + return portbinding + + def add_ha_port(self, context, router_id, network_id, tenant_id): + port = self._core_plugin.create_port(context, { + 'port': + {'tenant_id': '', + 'network_id': network_id, + 'fixed_ips': attributes.ATTR_NOT_SPECIFIED, + 'mac_address': attributes.ATTR_NOT_SPECIFIED, + 'admin_state_up': True, + 'device_id': router_id, + 'device_owner': constants.DEVICE_OWNER_ROUTER_HA_INTF, + 'name': constants.HA_PORT_NAME % tenant_id}}) + + try: + return self._create_ha_port_binding(context, port['id'], router_id) + except Exception: + with excutils.save_and_reraise_exception(): + self._core_plugin.delete_port(context, port['id'], + l3_port_check=False) + + def _create_ha_interfaces(self, context, router, ha_network): + admin_ctx = context.elevated() + + num_agents = self.get_number_of_agents_for_scheduling(context) + + port_ids = [] + try: + for index in range(num_agents): + binding = self.add_ha_port(admin_ctx, router.id, + ha_network.network['id'], + router.tenant_id) + port_ids.append(binding.port_id) + except Exception: + with excutils.save_and_reraise_exception(): + for port_id in port_ids: + self._core_plugin.delete_port(admin_ctx, port_id, + l3_port_check=False) + + def _delete_ha_interfaces(self, context, router_id): + admin_ctx = context.elevated() + device_filter = {'device_id': [router_id], + 'device_owner': + [constants.DEVICE_OWNER_ROUTER_HA_INTF]} + ports = self._core_plugin.get_ports(admin_ctx, filters=device_filter) + + for port in ports: + self._core_plugin.delete_port(admin_ctx, port['id'], + l3_port_check=False) + + def _notify_ha_interfaces_updated(self, context, router_id): + self.l3_rpc_notifier.routers_updated(context, [router_id]) + + @classmethod + def _is_ha(cls, router): + ha = router.get('ha') + if not attributes.is_attr_set(ha): + ha = cfg.CONF.l3_ha + return ha + + def _create_router_db(self, context, router, tenant_id): + router['ha'] = self._is_ha(router) + + if router['ha'] and l3_dvr_db.is_distributed_router(router): + raise l3_ha.DistributedHARouterNotSupported() + + with context.session.begin(subtransactions=True): + router_db = super(L3_HA_NAT_db_mixin, self)._create_router_db( + context, router, tenant_id) + + if router['ha']: + try: + ha_network = self.get_ha_network(context, + router_db.tenant_id) + if not ha_network: + ha_network = self._create_ha_network(context, + router_db.tenant_id) + + self._set_vr_id(context, router_db, ha_network) + self._create_ha_interfaces(context, router_db, ha_network) + self._notify_ha_interfaces_updated(context, router_db.id) + except Exception: + with excutils.save_and_reraise_exception(): + self.delete_router(context, router_db.id) + + return router_db + + def _update_router_db(self, context, router_id, data, gw_info): + ha = data.pop('ha', None) + + if ha and data.get('distributed'): + raise l3_ha.DistributedHARouterNotSupported() + + with context.session.begin(subtransactions=True): + router_db = super(L3_HA_NAT_db_mixin, self)._update_router_db( + context, router_id, data, gw_info) + + ha_not_changed = ha is None or ha == router_db.extra_attributes.ha + if ha_not_changed: + return router_db + + ha_network = self.get_ha_network(context, + router_db.tenant_id) + router_db.extra_attributes.ha = ha + if not ha: + self._delete_vr_id_allocation( + context, ha_network, router_db.extra_attributes.ha_vr_id) + router_db.extra_attributes.ha_vr_id = None + + if ha: + if not ha_network: + ha_network = self._create_ha_network(context, + router_db.tenant_id) + + self._set_vr_id(context, router_db, ha_network) + self._create_ha_interfaces(context, router_db, ha_network) + self._notify_ha_interfaces_updated(context, router_db.id) + else: + self._delete_ha_interfaces(context, router_db.id) + self._notify_ha_interfaces_updated(context, router_db.id) + + return router_db + + def update_router_state(self, context, router_id, state, host): + with context.session.begin(subtransactions=True): + bindings = self.get_ha_router_port_bindings(context, [router_id], + host) + if bindings: + if len(bindings) > 1: + LOG.warn(_LW("The router %(router_id)s is bound multiple " + "times on the agent %(host)s"), + {'router_id': router_id, 'host': host}) + + bindings[0].update({'state': state}) + + def delete_router(self, context, id): + router_db = self._get_router(context, id) + if router_db.extra_attributes.ha: + ha_network = self.get_ha_network(context, + router_db.tenant_id) + if ha_network: + self._delete_vr_id_allocation( + context, ha_network, router_db.extra_attributes.ha_vr_id) + self._delete_ha_interfaces(context, router_db.id) + + return super(L3_HA_NAT_db_mixin, self).delete_router(context, id) + + def get_ha_router_port_bindings(self, context, router_ids, host=None): + query = context.session.query(L3HARouterAgentPortBinding) + + if host: + query = query.join(agents_db.Agent).filter( + agents_db.Agent.host == host) + + query = query.filter( + L3HARouterAgentPortBinding.router_id.in_(router_ids)) + + return query.all() + + def _process_sync_ha_data(self, context, routers, host): + routers_dict = dict((router['id'], router) for router in routers) + + bindings = self.get_ha_router_port_bindings(context, + routers_dict.keys(), + host) + for binding in bindings: + port_dict = self._core_plugin._make_port_dict(binding.port) + + router = routers_dict.get(binding.router_id) + router[constants.HA_INTERFACE_KEY] = port_dict + router[constants.HA_ROUTER_STATE_KEY] = binding.state + + for router in routers_dict.values(): + interface = router.get(constants.HA_INTERFACE_KEY) + if interface: + self._populate_subnet_for_ports(context, [interface]) + + return routers_dict.values() + + def get_ha_sync_data_for_host(self, context, host=None, router_ids=None, + active=None): + sync_data = super(L3_HA_NAT_db_mixin, self).get_sync_data(context, + router_ids, + active) + return self._process_sync_ha_data(context, sync_data, host) diff --git a/neutron/db/migration/alembic_migrations/versions/16a27a58e093_ext_l3_ha_mode.py b/neutron/db/migration/alembic_migrations/versions/16a27a58e093_ext_l3_ha_mode.py new file mode 100644 index 000000000..cb1cb0458 --- /dev/null +++ b/neutron/db/migration/alembic_migrations/versions/16a27a58e093_ext_l3_ha_mode.py @@ -0,0 +1,86 @@ +# Copyright 2014 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +"""ext_l3_ha_mode + +Revision ID: 16a27a58e093 +Revises: 86d6d9776e2b +Create Date: 2014-02-01 10:24:12.412733 + +""" + +# revision identifiers, used by Alembic. +revision = '16a27a58e093' +down_revision = '86d6d9776e2b' + + +from alembic import op +import sqlalchemy as sa + +l3_ha_states = sa.Enum('active', 'standby', name='l3_ha_states') + + +def upgrade(active_plugins=None, options=None): + op.add_column('router_extra_attributes', + sa.Column('ha', sa.Boolean(), + nullable=False, + server_default=sa.sql.false())) + op.add_column('router_extra_attributes', + sa.Column('ha_vr_id', sa.Integer())) + + op.create_table('ha_router_agent_port_bindings', + sa.Column('port_id', sa.String(length=36), + nullable=False), + sa.Column('router_id', sa.String(length=36), + nullable=False), + sa.Column('l3_agent_id', sa.String(length=36), + nullable=True), + sa.Column('state', l3_ha_states, + server_default='standby'), + sa.PrimaryKeyConstraint('port_id'), + sa.ForeignKeyConstraint(['port_id'], ['ports.id'], + ondelete='CASCADE'), + sa.ForeignKeyConstraint(['router_id'], ['routers.id'], + ondelete='CASCADE'), + sa.ForeignKeyConstraint(['l3_agent_id'], ['agents.id'], + ondelete='CASCADE')) + + op.create_table('ha_router_networks', + sa.Column('tenant_id', sa.String(length=255), + nullable=False, primary_key=True), + sa.Column('network_id', sa.String(length=36), + nullable=False, + primary_key=True), + sa.ForeignKeyConstraint(['network_id'], ['networks.id'], + ondelete='CASCADE')) + + op.create_table('ha_router_vrid_allocations', + sa.Column('network_id', sa.String(length=36), + nullable=False, + primary_key=True), + sa.Column('vr_id', sa.Integer(), + nullable=False, + primary_key=True), + sa.ForeignKeyConstraint(['network_id'], ['networks.id'], + ondelete='CASCADE')) + + +def downgrade(active_plugins=None, options=None): + op.drop_table('ha_router_vrid_allocations') + op.drop_table('ha_router_networks') + op.drop_table('ha_router_agent_port_bindings') + l3_ha_states.drop(op.get_bind(), checkfirst=False) + op.drop_column('router_extra_attributes', 'ha_vr_id') + op.drop_column('router_extra_attributes', 'ha') diff --git a/neutron/db/migration/alembic_migrations/versions/HEAD b/neutron/db/migration/alembic_migrations/versions/HEAD index afbbf75d3..487d741ba 100644 --- a/neutron/db/migration/alembic_migrations/versions/HEAD +++ b/neutron/db/migration/alembic_migrations/versions/HEAD @@ -1 +1 @@ -86d6d9776e2b +16a27a58e093 diff --git a/neutron/db/migration/models/head.py b/neutron/db/migration/models/head.py index d15d3df79..47cb2630b 100644 --- a/neutron/db/migration/models/head.py +++ b/neutron/db/migration/models/head.py @@ -34,6 +34,7 @@ from neutron.db import l3_attrs_db # noqa from neutron.db import l3_db # noqa from neutron.db import l3_dvrscheduler_db # noqa from neutron.db import l3_gwmode_db # noqa +from neutron.db import l3_hamode_db # noqa from neutron.db.loadbalancer import loadbalancer_db # noqa from neutron.db.metering import metering_db # noqa from neutron.db import model_base diff --git a/neutron/extensions/l3_ext_ha_mode.py b/neutron/extensions/l3_ext_ha_mode.py new file mode 100644 index 000000000..f8487bb5b --- /dev/null +++ b/neutron/extensions/l3_ext_ha_mode.py @@ -0,0 +1,91 @@ +# Copyright (C) 2014 eNovance SAS +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +from neutron.api import extensions +from neutron.api.v2 import attributes +from neutron.common import constants +from neutron.common import exceptions + +HA_INFO = 'ha' +EXTENDED_ATTRIBUTES_2_0 = { + 'routers': { + HA_INFO: {'allow_post': True, 'allow_put': True, + 'default': attributes.ATTR_NOT_SPECIFIED, 'is_visible': True, + 'enforce_policy': True, + 'convert_to': attributes.convert_to_boolean_if_not_none} + } +} + + +class DistributedHARouterNotSupported(NotImplementedError): + message = _("Currenly distributed HA routers are " + "not supported.") + + +class MaxVRIDAllocationTriesReached(exceptions.NeutronException): + message = _("Failed to allocate a VRID in the network %(network_id)s " + "for the router %(router_id)s after %(max_tries)s tries.") + + +class NoVRIDAvailable(exceptions.Conflict): + message = _("No more Virtual Router Identifier (VRID) available when " + "creating router %(router_id)s. The limit of number " + "of HA Routers per tenant is 254.") + + +class HANetworkCIDRNotValid(exceptions.NeutronException): + message = _("The HA Network CIDR specified in the configuration file " + "isn't valid; %(cidr)s.") + + +class HANotEnoughAvailableAgents(exceptions.NeutronException): + message = _("Not enough l3 agents available to ensure HA. Minimum " + "required %(min_agents)s, available %(num_agents)s.") + + +class HAMinimumAgentsNumberNotValid(exceptions.NeutronException): + message = (_("min_l3_agents_per_router config parameter is not valid. " + "It has to be equal to or more than %s for HA.") % + constants.MINIMUM_AGENTS_FOR_HA) + + +class L3_ext_ha_mode(extensions.ExtensionDescriptor): + """Extension class supporting virtual router in HA mode.""" + + @classmethod + def get_name(cls): + return "HA Router extension" + + @classmethod + def get_alias(cls): + return constants.L3_HA_MODE_EXT_ALIAS + + @classmethod + def get_description(cls): + return "Add HA capability to routers." + + @classmethod + def get_namespace(cls): + return "" + + @classmethod + def get_updated(cls): + return "2014-04-26T00:00:00-00:00" + + def get_extended_resources(self, version): + if version == "2.0": + return EXTENDED_ATTRIBUTES_2_0 + else: + return {} diff --git a/neutron/services/l3_router/l3_router_plugin.py b/neutron/services/l3_router/l3_router_plugin.py index 15e0cbf4a..671db44bc 100644 --- a/neutron/services/l3_router/l3_router_plugin.py +++ b/neutron/services/l3_router/l3_router_plugin.py @@ -24,18 +24,18 @@ from neutron.common import rpc as n_rpc from neutron.common import topics from neutron.db import common_db_mixin from neutron.db import extraroute_db -from neutron.db import l3_dvr_db from neutron.db import l3_dvrscheduler_db from neutron.db import l3_gwmode_db +from neutron.db import l3_hamode_db from neutron.openstack.common import importutils from neutron.plugins.common import constants class L3RouterPlugin(common_db_mixin.CommonDbMixin, extraroute_db.ExtraRoute_db_mixin, - l3_dvr_db.L3_NAT_with_dvr_db_mixin, l3_gwmode_db.L3_NAT_db_mixin, - l3_dvrscheduler_db.L3_DVRsch_db_mixin): + l3_dvrscheduler_db.L3_DVRsch_db_mixin, + l3_hamode_db.L3_HA_NAT_db_mixin): """Implementation of the Neutron L3 Router Service Plugin. @@ -43,17 +43,19 @@ class L3RouterPlugin(common_db_mixin.CommonDbMixin, router and floatingip resources and manages associated request/response. All DB related work is implemented in classes - l3_db.L3_NAT_db_mixin, l3_dvr_db.L3_NAT_with_dvr_db_mixin, and - extraroute_db.ExtraRoute_db_mixin. + l3_db.L3_NAT_db_mixin, l3_hamode_db.L3_HA_NAT_db_mixin, + l3_dvr_db.L3_NAT_with_dvr_db_mixin, and extraroute_db.ExtraRoute_db_mixin. """ supported_extension_aliases = ["dvr", "router", "ext-gw-mode", - "extraroute", "l3_agent_scheduler"] + "extraroute", "l3_agent_scheduler", + "l3-ha"] def __init__(self): self.setup_rpc() self.router_scheduler = importutils.import_object( cfg.CONF.router_scheduler_driver) self.start_periodic_agent_status_check() + super(L3RouterPlugin, self).__init__() def setup_rpc(self): # RPC support diff --git a/neutron/tests/unit/db/test_l3_dvr_db.py b/neutron/tests/unit/db/test_l3_dvr_db.py index 9612aa7b2..be27ce9ca 100644 --- a/neutron/tests/unit/db/test_l3_dvr_db.py +++ b/neutron/tests/unit/db/test_l3_dvr_db.py @@ -118,7 +118,7 @@ class L3DvrTestCase(testlib_api.SqlTestCase): pass_router_id=False) def _test__is_distributed_router(self, router, expected): - result = l3_dvr_db._is_distributed_router(router) + result = l3_dvr_db.is_distributed_router(router) self.assertEqual(expected, result) def test__is_distributed_router_by_db_object(self): diff --git a/neutron/tests/unit/db/test_l3_ha_db.py b/neutron/tests/unit/db/test_l3_ha_db.py new file mode 100644 index 000000000..4616612bb --- /dev/null +++ b/neutron/tests/unit/db/test_l3_ha_db.py @@ -0,0 +1,390 @@ +# Copyright (C) 2014 eNovance SAS +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock +from oslo.config import cfg + +from neutron.common import constants +from neutron import context +from neutron.db import agents_db +from neutron.db import common_db_mixin +from neutron.db import l3_hamode_db +from neutron.extensions import l3_ext_ha_mode +from neutron import manager +from neutron.openstack.common import uuidutils +from neutron.tests.unit import testlib_api +from neutron.tests.unit import testlib_plugin + +_uuid = uuidutils.generate_uuid + + +class FakeL3Plugin(common_db_mixin.CommonDbMixin, + l3_hamode_db.L3_HA_NAT_db_mixin): + pass + + +class FakeL3PluginWithAgents(FakeL3Plugin, + agents_db.AgentDbMixin): + pass + + +class L3HATestFramework(testlib_api.SqlTestCase, + testlib_plugin.PluginSetupHelper): + def setUp(self): + super(L3HATestFramework, self).setUp() + + self.admin_ctx = context.get_admin_context() + self.setup_coreplugin('neutron.plugins.ml2.plugin.Ml2Plugin') + self.core_plugin = manager.NeutronManager.get_plugin() + mock.patch.object(l3_hamode_db.L3_HA_NAT_db_mixin, 'get_l3_agents', + create=True, return_value=[1, 2]).start() + notif_p = mock.patch.object(l3_hamode_db.L3_HA_NAT_db_mixin, + '_notify_ha_interfaces_updated') + self.notif_m = notif_p.start() + cfg.CONF.set_override('allow_overlapping_ips', True) + + def _create_router(self, ha=True, tenant_id='tenant1', distributed=None): + router = {'name': 'router1', 'admin_state_up': True} + if ha is not None: + router['ha'] = ha + if distributed is not None: + router['distributed'] = distributed + return self.plugin._create_router_db(self.admin_ctx, router, tenant_id) + + def _update_router(self, router_id, ha=True, distributed=None): + data = {'ha': ha} if ha is not None else {} + if distributed is not None: + data['distributed'] = distributed + return self.plugin._update_router_db(self.admin_ctx, router_id, + data, None) + + +class L3HAGetSyncDataTestCase(L3HATestFramework): + + def setUp(self): + super(L3HAGetSyncDataTestCase, self).setUp() + self.plugin = FakeL3PluginWithAgents() + self._register_agents() + + def _register_agents(self): + agent_status = { + 'agent_type': constants.AGENT_TYPE_L3, + 'binary': 'neutron-l3-agent', + 'host': 'l3host', + 'topic': 'N/A' + } + self.plugin.create_or_update_agent(self.admin_ctx, agent_status) + agent_status['host'] = 'l3host_2' + self.plugin.create_or_update_agent(self.admin_ctx, agent_status) + self.agent1, self.agent2 = self.plugin.get_agents(self.admin_ctx) + + def _bind_router(self, router_id): + with self.admin_ctx.session.begin(subtransactions=True): + bindings = self.plugin.get_ha_router_port_bindings(self.admin_ctx, + [router_id]) + + for agent_id, binding in zip( + [self.agent1['id'], self.agent2['id']], bindings): + binding.l3_agent_id = agent_id + + def test_l3_agent_routers_query_interface(self): + router = self._create_router() + self._bind_router(router.id) + routers = self.plugin.get_ha_sync_data_for_host(self.admin_ctx, + self.agent1['host']) + self.assertEqual(1, len(routers)) + router = routers[0] + + self.assertIsNotNone(router.get('ha')) + + interface = router.get(constants.HA_INTERFACE_KEY) + self.assertIsNotNone(interface) + + self.assertEqual(constants.DEVICE_OWNER_ROUTER_HA_INTF, + interface['device_owner']) + self.assertEqual(cfg.CONF.l3_ha_net_cidr, interface['subnet']['cidr']) + + def test_update_state(self): + router = self._create_router() + self._bind_router(router.id) + routers = self.plugin.get_ha_sync_data_for_host(self.admin_ctx, + self.agent1['host']) + state = routers[0].get(constants.HA_ROUTER_STATE_KEY) + self.assertEqual('standby', state) + + self.plugin.update_router_state(self.admin_ctx, router.id, 'active', + self.agent1['host']) + + routers = self.plugin.get_ha_sync_data_for_host(self.admin_ctx, + self.agent1['host']) + + state = routers[0].get(constants.HA_ROUTER_STATE_KEY) + self.assertEqual('active', state) + + +class L3HATestCase(L3HATestFramework): + + def setUp(self): + super(L3HATestCase, self).setUp() + self.plugin = FakeL3Plugin() + + def test_verify_configuration_succeed(self): + # Default configuration should pass + self.plugin._verify_configuration() + + def test_verify_configuration_l3_ha_net_cidr_is_not_a_cidr(self): + cfg.CONF.set_override('l3_ha_net_cidr', 'not a cidr') + self.assertRaises( + l3_ext_ha_mode.HANetworkCIDRNotValid, + self.plugin._verify_configuration) + + def test_verify_configuration_l3_ha_net_cidr_is_not_a_subnet(self): + cfg.CONF.set_override('l3_ha_net_cidr', '10.0.0.1/8') + self.assertRaises( + l3_ext_ha_mode.HANetworkCIDRNotValid, + self.plugin._verify_configuration) + + def test_verify_conifguration_min_l3_agents_per_router_below_minimum(self): + cfg.CONF.set_override('min_l3_agents_per_router', 0) + self.assertRaises( + l3_ext_ha_mode.HAMinimumAgentsNumberNotValid, + self.plugin._verify_configuration) + + def test_ha_router_create(self): + router = self._create_router() + self.assertTrue(router.extra_attributes['ha']) + + def test_ha_router_create_with_distributed(self): + self.assertRaises(l3_ext_ha_mode.DistributedHARouterNotSupported, + self._create_router, + distributed=True) + + def test_no_ha_router_create(self): + router = self._create_router(ha=False) + self.assertFalse(router.extra_attributes['ha']) + + def test_router_create_with_ha_conf_enabled(self): + cfg.CONF.set_override('l3_ha', True) + + router = self._create_router(ha=None) + self.assertTrue(router.extra_attributes['ha']) + + def test_migration_from_ha(self): + router = self._create_router() + self.assertTrue(router.extra_attributes['ha']) + + router = self._update_router(router.id, ha=False) + self.assertFalse(router.extra_attributes['ha']) + self.assertIsNone(router.extra_attributes['ha_vr_id']) + + def test_migration_to_ha(self): + router = self._create_router(ha=False) + self.assertFalse(router.extra_attributes['ha']) + + router = self._update_router(router.id, ha=True) + self.assertTrue(router.extra_attributes['ha']) + self.assertIsNotNone(router.extra_attributes['ha_vr_id']) + + def test_migrate_ha_router_to_distributed(self): + router = self._create_router() + self.assertTrue(router.extra_attributes['ha']) + + self.assertRaises(l3_ext_ha_mode.DistributedHARouterNotSupported, + self._update_router, + router.id, + distributed=True) + + def test_unique_ha_network_per_tenant(self): + tenant1 = _uuid() + tenant2 = _uuid() + self._create_router(tenant_id=tenant1) + self._create_router(tenant_id=tenant2) + ha_network1 = self.plugin.get_ha_network(self.admin_ctx, tenant1) + ha_network2 = self.plugin.get_ha_network(self.admin_ctx, tenant2) + self.assertNotEqual( + ha_network1['network_id'], ha_network2['network_id']) + + def _deployed_router_change_ha_flag(self, to_ha): + self._create_router(ha=not to_ha) + routers = self.plugin.get_ha_sync_data_for_host(self.admin_ctx) + router = routers[0] + interface = router.get(constants.HA_INTERFACE_KEY) + if to_ha: + self.assertIsNone(interface) + else: + self.assertIsNotNone(interface) + + self._update_router(router['id'], to_ha) + routers = self.plugin.get_ha_sync_data_for_host(self.admin_ctx) + router = routers[0] + interface = router.get(constants.HA_INTERFACE_KEY) + if to_ha: + self.assertIsNotNone(interface) + else: + self.assertIsNone(interface) + + def test_deployed_router_can_have_ha_enabled(self): + self._deployed_router_change_ha_flag(to_ha=True) + + def test_deployed_router_can_have_ha_disabled(self): + self._deployed_router_change_ha_flag(to_ha=False) + + def test_create_ha_router_notifies_agent(self): + self._create_router() + self.assertTrue(self.notif_m.called) + + def test_update_router_to_ha_notifies_agent(self): + router = self._create_router(ha=False) + self.notif_m.reset_mock() + self._update_router(router.id, ha=True) + self.assertTrue(self.notif_m.called) + + def test_unique_vr_id_between_routers(self): + self._create_router() + self._create_router() + routers = self.plugin.get_ha_sync_data_for_host(self.admin_ctx) + self.assertEqual(2, len(routers)) + self.assertNotEqual(routers[0]['ha_vr_id'], routers[1]['ha_vr_id']) + + @mock.patch('neutron.db.l3_hamode_db.VR_ID_RANGE', new=set(range(1, 1))) + def test_vr_id_depleted(self): + self.assertRaises(l3_ext_ha_mode.NoVRIDAvailable, self._create_router) + + @mock.patch('neutron.db.l3_hamode_db.VR_ID_RANGE', new=set(range(1, 2))) + def test_vr_id_unique_range_per_tenant(self): + self._create_router() + self._create_router(tenant_id=_uuid()) + routers = self.plugin.get_ha_sync_data_for_host(self.admin_ctx) + self.assertEqual(2, len(routers)) + self.assertEqual(routers[0]['ha_vr_id'], routers[1]['ha_vr_id']) + + @mock.patch('neutron.db.l3_hamode_db.MAX_ALLOCATION_TRIES', new=2) + def test_vr_id_allocation_contraint_conflict(self): + router = self._create_router() + network = self.plugin.get_ha_network(self.admin_ctx, router.tenant_id) + + with mock.patch.object(self.plugin, '_get_allocated_vr_id', + return_value=set()) as alloc: + self.assertRaises(l3_ext_ha_mode.MaxVRIDAllocationTriesReached, + self.plugin._allocate_vr_id, self.admin_ctx, + network.network_id, router.id) + self.assertEqual(2, len(alloc.mock_calls)) + + def test_vr_id_allocation_delete_router(self): + router = self._create_router() + network = self.plugin.get_ha_network(self.admin_ctx, router.tenant_id) + + allocs_before = self.plugin._get_allocated_vr_id(self.admin_ctx, + network.network_id) + router = self._create_router() + allocs_current = self.plugin._get_allocated_vr_id(self.admin_ctx, + network.network_id) + self.assertNotEqual(allocs_before, allocs_current) + + self.plugin.delete_router(self.admin_ctx, router.id) + allocs_after = self.plugin._get_allocated_vr_id(self.admin_ctx, + network.network_id) + self.assertEqual(allocs_before, allocs_after) + + def test_vr_id_allocation_router_migration(self): + router = self._create_router() + network = self.plugin.get_ha_network(self.admin_ctx, router.tenant_id) + + allocs_before = self.plugin._get_allocated_vr_id(self.admin_ctx, + network.network_id) + router = self._create_router() + self._update_router(router.id, ha=False) + allocs_after = self.plugin._get_allocated_vr_id(self.admin_ctx, + network.network_id) + self.assertEqual(allocs_before, allocs_after) + + def test_one_ha_router_one_not(self): + self._create_router(ha=False) + self._create_router() + routers = self.plugin.get_ha_sync_data_for_host(self.admin_ctx) + + ha0 = routers[0]['ha'] + ha1 = routers[1]['ha'] + + self.assertNotEqual(ha0, ha1) + + def test_add_ha_port_binding_failure_rolls_back_port(self): + router = self._create_router() + device_filter = {'device_id': [router.id]} + ports_before = self.core_plugin.get_ports( + self.admin_ctx, filters=device_filter) + network = self.plugin.get_ha_network(self.admin_ctx, router.tenant_id) + + with mock.patch.object(self.plugin, '_create_ha_port_binding', + side_effect=ValueError): + self.assertRaises(ValueError, self.plugin.add_ha_port, + self.admin_ctx, router.id, network.network_id, + router.tenant_id) + + ports_after = self.core_plugin.get_ports( + self.admin_ctx, filters=device_filter) + + self.assertEqual(ports_before, ports_after) + + def test_create_ha_network_binding_failure_rolls_back_network(self): + networks_before = self.core_plugin.get_networks(self.admin_ctx) + + with mock.patch.object(self.plugin, + '_create_ha_network_tenant_binding', + side_effect=ValueError): + self.assertRaises(ValueError, self.plugin._create_ha_network, + self.admin_ctx, _uuid()) + + networks_after = self.core_plugin.get_networks(self.admin_ctx) + self.assertEqual(networks_before, networks_after) + + def test_create_ha_network_subnet_failure_rolls_back_network(self): + networks_before = self.core_plugin.get_networks(self.admin_ctx) + + with mock.patch.object(self.plugin, '_create_ha_subnet', + side_effect=ValueError): + self.assertRaises(ValueError, self.plugin._create_ha_network, + self.admin_ctx, _uuid()) + + networks_after = self.core_plugin.get_networks(self.admin_ctx) + self.assertEqual(networks_before, networks_after) + + def test_create_ha_interfaces_binding_failure_rolls_back_ports(self): + router = self._create_router() + network = self.plugin.get_ha_network(self.admin_ctx, router.tenant_id) + device_filter = {'device_id': [router.id]} + ports_before = self.core_plugin.get_ports( + self.admin_ctx, filters=device_filter) + + with mock.patch.object(self.plugin, '_create_ha_port_binding', + side_effect=ValueError): + self.assertRaises(ValueError, self.plugin._create_ha_interfaces, + self.admin_ctx, router, network) + + ports_after = self.core_plugin.get_ports( + self.admin_ctx, filters=device_filter) + self.assertEqual(ports_before, ports_after) + + def test_create_router_db_ha_attribute_failure_rolls_back_router(self): + routers_before = self.plugin.get_routers(self.admin_ctx) + + for method in ('_set_vr_id', + '_create_ha_interfaces', + '_notify_ha_interfaces_updated'): + with mock.patch.object(self.plugin, method, + side_effect=ValueError): + self.assertRaises(ValueError, self._create_router) + + routers_after = self.plugin.get_routers(self.admin_ctx) + self.assertEqual(routers_before, routers_after)