Add no nat cidrs network extension

The no-NAT CIDRs extension is applied to the network resource
in neutron. When applied, it affects the list of subnets that
should be reachable without NAT that are delivered in the RPC
calls to agents. The agents can then use this information to
ensure that specific destination CIDRs will never use NAT.

The extension can be applied to both tenant and external/public
networks. The extension should be used judiciously, as placing
it on a network will cause those CIDRs to be added to all RPC
calls requesting subnets within that VRF (e.g. the extension
could be added to a shared network or to a network that uses
a subnetpool relating to a shared address scope, which would
be seen by all other networks that report to that same address
scope or shared network).

Change-Id: Ic2cdd501933cc21c286ca36218361aadef1878b8

gbpservice/neutron/db/migration/alembic_migrations/versions/HEAD

@@ -1 +1 @@
-90e1d92a49b2
+e29a84f6a15f

gbpservice/neutron/db/migration/alembic_migrations/versions/e29a84f6a15f_add_no_nat_cidrs_network_extension.py

@@ -0,0 +1,42 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+
+"""add no nat cidrs network extension
+
+Revision ID: e29a84f6a15f
+Revises: 90e1d92a49b2
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'e29a84f6a15f'
+down_revision = '90e1d92a49b2'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    op.create_table(
+        'apic_aim_network_no_nat_cidrs',
+        sa.Column('network_id', sa.String(36), nullable=False),
+        sa.Column('cidr', sa.String(64), nullable=False),
+        sa.ForeignKeyConstraint(['network_id'], ['networks.id'],
+                        name='apic_aim_network_no_nat_cidrs_extn_fk_network',
+                        ondelete='CASCADE'),
+        sa.PrimaryKeyConstraint('network_id', 'cidr')
+    )
+
+
+def downgrade():
+    pass

gbpservice/neutron/extensions/cisco_apic.py

@@ -53,6 +53,7 @@ ERSPAN_CONFIG = 'apic:erspan_config'
 POLICY_ENFORCEMENT_PREF = 'apic:policy_enforcement_pref'
 SNAT_SUBNET_ONLY = 'apic:snat_subnet_only'
 EPG_SUBNET = 'apic:epg_subnet'
+NO_NAT_CIDRS = 'apic:no_nat_cidrs'
 
 BD = 'BridgeDomain'
 EPG = 'EndpointGroup'
@@ -354,6 +355,12 @@ NET_ATTRIBUTES = {
         'is_visible': True, 'default': 'unenforced',
         'validate': {'type:values': ['unenforced', 'enforced', '']},
     },
+    NO_NAT_CIDRS: {
+        'allow_post': True, 'allow_put': True,
+        'is_visible': True, 'default': None,
+        'convert_to': convert_apic_none_to_empty_list,
+        'validate': {'type:list_of_unique_strings': None},
+    },
 }
 
 EXT_NET_ATTRIBUTES = {

gbpservice/neutron/plugins/ml2plus/drivers/apic_aim/extension_db.py

@@ -140,6 +140,21 @@ class NetworkExtEpgContractMasterDb(model_base.BASEV2):
                                    lazy='joined', cascade='delete'))
 
 
+class NetworkExtensionNoNatCidrsDb(model_base.BASEV2):
+
+    __tablename__ = 'apic_aim_network_no_nat_cidrs'
+
+    network_id = sa.Column(
+        sa.String(36), sa.ForeignKey('networks.id', ondelete="CASCADE"),
+        primary_key=True)
+    cidr = sa.Column(sa.String(64), primary_key=True)
+    network = orm.relationship(models_v2.Network,
+                               backref=orm.backref(
+                                   'aim_extension_no_nat_cidrs_mapping',
+                                   lazy='joined', uselist=True,
+                                   cascade='delete'))
+
+
 class SubnetExtensionDb(model_base.BASEV2):
 
     __tablename__ = 'apic_aim_subnet_extensions'
@@ -275,10 +290,19 @@ class ExtensionDbMixin(object):
         db_masters = query(session).params(
             network_ids=network_ids).all()
 
+        query = BAKERY(lambda s: s.query(
+            NetworkExtensionNoNatCidrsDb))
+        query += lambda q: q.filter(
+            NetworkExtensionNoNatCidrsDb.network_id.in_(
+                sa.bindparam('network_ids', expanding=True)))
+        db_no_nat_cidrs = query(session).params(
+            network_ids=network_ids).all()
+
         cidrs_by_net_id = {}
         vlans_by_net_id = {}
         contracts_by_net_id = {}
         masters_by_net_id = {}
+        no_nat_cidrs_by_net_id = {}
         for db_cidr in db_cidrs:
             cidrs_by_net_id.setdefault(db_cidr.network_id, []).append(
                 db_cidr)
@@ -291,6 +315,9 @@ class ExtensionDbMixin(object):
         for db_master in db_masters:
             masters_by_net_id.setdefault(db_master.network_id, []).append(
                 db_master)
+        for db_no_nat_cidr in db_no_nat_cidrs:
+            no_nat_cidrs_by_net_id.setdefault(db_no_nat_cidr.network_id,
+            []).append(db_no_nat_cidr)
 
         result = {}
         for db_obj in db_objs:
@@ -299,11 +326,13 @@ class ExtensionDbMixin(object):
                 db_obj, cidrs_by_net_id.get(net_id, []),
                 vlans_by_net_id.get(net_id, []),
                 contracts_by_net_id.get(net_id, []),
-                masters_by_net_id.get(net_id, [])))
+                masters_by_net_id.get(net_id, []),
+                no_nat_cidrs_by_net_id.get(net_id, [])))
         return result
 
     def make_network_extn_db_conf_dict(self, ext_db, db_cidrs, db_vlans,
-                                       db_contracts, db_masters):
+                                       db_contracts, db_masters,
+                                       db_no_nat_cidrs):
         net_res = {}
         db_obj = ext_db
         if db_obj:
@@ -338,6 +367,8 @@ class ExtensionDbMixin(object):
                  'name': m.name} for m in db_masters]
             net_res[cisco_apic.POLICY_ENFORCEMENT_PREF] = db_obj[
                 'policy_enforcement_pref']
+            net_res[cisco_apic.NO_NAT_CIDRS] = [
+                c.cidr for c in db_no_nat_cidrs]
         if net_res.get(cisco_apic.EXTERNAL_NETWORK):
             net_res[cisco_apic.EXTERNAL_CIDRS] = [c.cidr for c in db_cidrs]
         return net_res
@@ -418,6 +449,12 @@ class ExtensionDbMixin(object):
                         res_dict[cisco_apic.EPG_CONTRACT_MASTERS],
                         network_id=network_id)
 
+            if cisco_apic.NO_NAT_CIDRS in res_dict:
+                self._update_list_attr(session, NetworkExtensionNoNatCidrsDb,
+                                       'cidr',
+                                       res_dict[cisco_apic.NO_NAT_CIDRS],
+                                       network_id=network_id)
+
     def get_network_ids_by_ext_net_dn(self, session, dn, lock_update=False):
         query = BAKERY(lambda s: s.query(
             NetworkExtensionDb.network_id))

gbpservice/neutron/plugins/ml2plus/drivers/apic_aim/extension_driver.py

@@ -160,6 +160,8 @@ class ApicExtensionDriver(api_plus.ExtensionDriver,
                     data.get(cisco_apic.EPG_CONTRACT_MASTERS),
                     cisco_apic.POLICY_ENFORCEMENT_PREF:
                     data.get(cisco_apic.POLICY_ENFORCEMENT_PREF, "unenforced"),
+                    cisco_apic.NO_NAT_CIDRS:
+                    data.get(cisco_apic.NO_NAT_CIDRS),
                     }
         if cisco_apic.VLANS_LIST in (data.get(
                 cisco_apic.NESTED_DOMAIN_ALLOWED_VLANS) or {}):
@@ -222,7 +224,8 @@ class ApicExtensionDriver(api_plus.ExtensionDriver,
                 cisco_apic.EXTRA_PROVIDED_CONTRACTS,
                 cisco_apic.EXTRA_CONSUMED_CONTRACTS,
                 cisco_apic.EPG_CONTRACT_MASTERS,
-                cisco_apic.POLICY_ENFORCEMENT_PREF]
+                cisco_apic.POLICY_ENFORCEMENT_PREF,
+                cisco_apic.NO_NAT_CIDRS]
         if not(set(update_attrs) & set(data.keys())):
             return
 
@@ -242,7 +245,8 @@ class ApicExtensionDriver(api_plus.ExtensionDriver,
                 cisco_apic.EXTRA_PROVIDED_CONTRACTS,
                 cisco_apic.EXTRA_CONSUMED_CONTRACTS,
                 cisco_apic.EPG_CONTRACT_MASTERS,
-                cisco_apic.POLICY_ENFORCEMENT_PREF]
+                cisco_apic.POLICY_ENFORCEMENT_PREF,
+                cisco_apic.NO_NAT_CIDRS]
         for e_k in ext_keys:
             if e_k in data:
                 res_dict.update({e_k: data[e_k]})

gbpservice/neutron/plugins/ml2plus/drivers/apic_aim/mechanism_driver.py

@@ -1422,7 +1422,8 @@ class ApicMechanismDriver(api_plus.MechanismDriver,
                     net_db.aim_extension_cidr_mapping,
                     net_db.aim_extension_domain_mapping,
                     net_db.aim_extension_extra_contract_mapping,
-                    net_db.aim_extension_epg_contract_masters)
+                    net_db.aim_extension_epg_contract_masters,
+                    net_db.aim_extension_no_nat_cidrs_mapping)
             if cisco_apic.EXTERNAL_NETWORK in ext_dict:
                 dn = ext_dict.pop(cisco_apic.EXTERNAL_NETWORK)
                 a_ext_net = aim_resource.ExternalNetwork.from_dn(dn)

gbpservice/neutron/plugins/ml2plus/drivers/apic_aim/rpc.py

@@ -455,7 +455,8 @@ class ApicRpcHandlerMixin(object):
 
                     # Query for VRF subnets.
                     info['vrf_subnets'] = self._query_vrf_subnets(
-                        session, port_info.vrf_tenant_name, port_info.vrf_name)
+                        session, port_info.vrf_tenant_name, port_info.vrf_name,
+                        ext_net_info=info['ext_net_info'])
 
                     # Let the GBP policy driver do its queries and add
                     # its info.
@@ -849,14 +850,20 @@ class ApicRpcHandlerMixin(object):
         return [x for x, in query(session).params(
             network_id=network_id)]
 
-    def _query_vrf_subnets(self, session, vrf_tenant_name, vrf_name):
+    def _query_vrf_subnets(self, session, vrf_tenant_name, vrf_name,
+                           ext_net_info=None):
         # A VRF mapped from one or two (IPv4 and/or IPv6)
         # address_scopes cannot be associated with unscoped
         # subnets. So first see if the VRF is mapped from
         # address_scopes, and if so, return the subnetpool CIDRs
         # associated with those address_scopes.
+        result = []
+        sub_ids = []
+        net_ids = []
         query = BAKERY(lambda s: s.query(
-            models_v2.SubnetPoolPrefix.cidr))
+            models_v2.SubnetPoolPrefix.cidr,
+            models_v2.Subnet.id,
+            models_v2.Network.id))
         query += lambda q: q.join(
             models_v2.SubnetPool,
             models_v2.SubnetPool.id ==
@@ -865,15 +872,37 @@ class ApicRpcHandlerMixin(object):
             db.AddressScopeMapping,
             db.AddressScopeMapping.scope_id ==
             models_v2.SubnetPool.address_scope_id)
+        query += lambda q: q.join(
+            db.NetworkMapping,
+            db.NetworkMapping.network_id ==
+            models_v2.Network.id)
+        query += lambda q: q.join(
+            models_v2.Subnet,
+            models_v2.Subnet.network_id ==
+            models_v2.Network.id)
         query += lambda q: q.filter(
             db.AddressScopeMapping.vrf_name ==
             sa.bindparam('vrf_name'),
             db.AddressScopeMapping.vrf_tenant_name ==
+            sa.bindparam('vrf_tenant_name'),
+            db.NetworkMapping.vrf_name ==
+            sa.bindparam('vrf_name'),
+            db.NetworkMapping.vrf_tenant_name ==
             sa.bindparam('vrf_tenant_name'))
-        result = [x for x, in query(session).params(
-            vrf_name=vrf_name,
-            vrf_tenant_name=vrf_tenant_name)]
+        for cidr, sub_id, net_id in query(session).params(vrf_name=vrf_name,
+                        vrf_tenant_name=vrf_tenant_name).all():
+            result.append(cidr)
+            sub_ids.append(sub_id)
+            net_ids.append(net_id)
         if result:
+            # query to fetch no nat cidrs extension from the networks
+            if not ext_net_info:
+                ext_net_info = self._query_endpoint_ext_net_info(
+                    session, sub_ids)
+            net_ids.extend(list(ext_net_info.keys()))
+            cidrs = self._query_no_nat_cidrs_extension(session, net_ids)
+            if cidrs:
+                result.extend(cidrs)
             return result
 
         # If the VRF is not mapped from address_scopes, return the
@@ -886,7 +915,9 @@ class ApicRpcHandlerMixin(object):
         # subnets' CIDRs being returned, even for the scoped case
         # where they are not needed, so it may not be a win.
         query = BAKERY(lambda s: s.query(
-            models_v2.Subnet.cidr))
+            models_v2.Subnet.cidr,
+            models_v2.Subnet.id,
+            models_v2.Subnet.network_id))
         query += lambda q: q.join(
             db.NetworkMapping,
             db.NetworkMapping.network_id ==
@@ -896,9 +927,35 @@ class ApicRpcHandlerMixin(object):
             sa.bindparam('vrf_name'),
             db.NetworkMapping.vrf_tenant_name ==
             sa.bindparam('vrf_tenant_name'))
-        return [x for x, in query(session).params(
-            vrf_name=vrf_name,
-            vrf_tenant_name=vrf_tenant_name)]
+        for cidr, sub_id, net_id in query(session).params(vrf_name=vrf_name,
+                        vrf_tenant_name=vrf_tenant_name).all():
+            result.append(cidr)
+            sub_ids.append(sub_id)
+            net_ids.append(net_id)
+        if not ext_net_info:
+            ext_net_info = self._query_endpoint_ext_net_info(
+                session, sub_ids)
+        net_ids.extend(list(ext_net_info.keys()))
+        # query to fetch no nat cidrs extension from the networks
+        cidrs = self._query_no_nat_cidrs_extension(session, net_ids)
+        if cidrs:
+            result.extend(cidrs)
+        return result
+
+    def _query_no_nat_cidrs_extension(self, session, net_ids):
+        query = BAKERY(lambda s: s.query(
+            extension_db.NetworkExtensionNoNatCidrsDb.cidr))
+        query += lambda q: q.join(
+            models_v2.Network,
+            models_v2.Network.id ==
+            extension_db.NetworkExtensionNoNatCidrsDb.network_id)
+        query += lambda q: q.filter(
+            extension_db.NetworkExtensionNoNatCidrsDb.network_id.in_(
+                sa.bindparam('network_ids', expanding=True)))
+        query += lambda q: q.distinct()
+        cidrs = [x for x, in query(session).params(
+                network_ids=net_ids)]
+        return cidrs
 
     def _build_endpoint_neutron_details(self, info):
         port_info = info['port_info']

gbpservice/neutron/tests/unit/plugins/ml2plus/test_apic_aim.py

@@ -7133,6 +7133,49 @@ class TestExtensionAttributes(ApicAimTestCase):
                 network_id=net_id).all())
         self.assertEqual([], db_masters)
 
+    def test_network_with_no_nat_cidrs_lifecycle(self):
+        session = db_api.get_reader_session()
+        extn = extn_db.ExtensionDbMixin()
+
+        # Create network with default no nat cidrs extension
+        net = self._make_network(
+            self.fmt, 'net1', True)['network']
+        net_id = net['id']
+        self.assertItemsEqual([],
+            net['apic:no_nat_cidrs'])
+
+        # Test show.
+        net = self._show('networks', net_id)['network']
+        self.assertItemsEqual([],
+            net['apic:no_nat_cidrs'])
+
+        # Test list.
+        net = self._list(
+            'networks', query_params=('id=%s' % net_id))['networks'][0]
+        self.assertItemsEqual([],
+            net['apic:no_nat_cidrs'])
+
+        # Test update with no nat cidrs extension
+        net = self._update(
+            'networks', net_id,
+            {'network':
+             {'apic:no_nat_cidrs': ['10.10.10.0/24']}})['network']
+        self.assertItemsEqual(['10.10.10.0/24'],
+            net['apic:no_nat_cidrs'])
+
+        # Test show after update.
+        net = self._show('networks', net_id)['network']
+        self.assertItemsEqual(['10.10.10.0/24'],
+            net['apic:no_nat_cidrs'])
+
+        # Test delete.
+        self._delete('networks', net_id)
+        self.assertFalse(extn.get_network_extn_db(session, net_id))
+        db_masters = (session.query(
+            extn_db.NetworkExtensionNoNatCidrsDb).filter_by(
+                network_id=net_id).all())
+        self.assertEqual([], db_masters)
+
     def test_external_network_lifecycle(self):
         session = db_api.get_reader_session()
         extn = extn_db.ExtensionDbMixin()
@@ -12776,3 +12819,340 @@ class TestUpdateRouterSubnet(ApicAimTestCase):
         self._check_ip_in_cidr(router
             ['external_gateway_info']['external_fixed_ips'][0]['ip_address'],
             fip_sub['cidr'])
+
+
+class TestExtensionNoNatCidrsScenarios(TestOpflexRpc, ApicAimTestCase):
+    def _get_vrfid_from_net(self, net):
+        kwargs = {}
+        vrf_dn = net['network']['apic:distinguished_names']['VRF']
+        _, tenant_rn, vrf_rn = vrf_dn.split('/')
+        tenant = tenant_rn.split('tn-')[1]
+        vrf = vrf_rn.split('ctx-')[1]
+        kwargs['vrf_id'] = '%s %s' % (tenant, vrf)
+        return kwargs
+
+    def test_no_nat_cidrs_isolated_private_net(self):
+        # test no nat cidrs extension for isolated private network
+        host = 'host1'
+        kwargs = {'apic:no_nat_cidrs': ['10.10.10.0/24']}
+        net = self._make_network(
+            self.fmt, 'net10', True,
+            arg_list=tuple(list(kwargs.keys())), **kwargs)
+        net_id = net['network']['id']
+
+        self._make_subnet(self.fmt, net, '10.0.1.1', '10.0.1.0/24')
+
+        port = self._make_port(self.fmt, net_id)['port']
+        port_id = port['id']
+        port = self._bind_port_to_host(port_id, host)['port']
+        self.assertEqual('ovs', port['binding:vif_type'])
+
+        # Call the RPC handler.
+        request = {
+            'device': 'tap' + port_id,
+            'timestamp': 12345,
+            'request_id': 'a_request'
+        }
+        response = self.driver.request_endpoint_details(
+            n_context.get_admin_context(), request=request, host=host)
+        self.assertEqual(set(['10.0.1.0/24', '10.10.10.0/24']),
+            set(response['gbp_details']['vrf_subnets']))
+        kwargs = self._get_vrfid_from_net(net)
+        response = self.driver.get_vrf_details(
+            n_context.get_admin_context(), **kwargs)
+        self.assertEqual(set(['10.0.1.0/24', '10.10.10.0/24']),
+            set(response['vrf_subnets']))
+
+    def test_no_nat_cidrs_external_network(self):
+        # test no nat cidrs extension for external network
+        ext_net = self._make_ext_network('ext-net1',
+                                         dn=self.dn_t1_l1_n1)
+        ext_net = self._update(
+            'networks', ext_net['id'],
+            {'network':
+             {'apic:no_nat_cidrs': ['50.50.50.0/24']}})
+        self._make_subnet(
+            self.fmt, {'network': ext_net['network']}, '100.100.100.1',
+            '100.100.100.0/24')
+
+        host = 'host1'
+        port = self._make_port(self.fmt, ext_net['network']['id'])['port']
+        port_id = port['id']
+        port = self._bind_port_to_host(port_id, host)['port']
+        self.assertEqual('ovs', port['binding:vif_type'])
+
+        # Call the RPC handler.
+        request = {
+            'device': 'tap' + port_id,
+            'timestamp': 12345,
+            'request_id': 'a_request'
+        }
+        response = self.driver.request_endpoint_details(
+            n_context.get_admin_context(), request=request, host=host)
+        self.assertEqual(['100.100.100.0/24', '50.50.50.0/24'],
+            response['gbp_details']['vrf_subnets'])
+        net = self._show('networks', ext_net['network']['id'])
+        kwargs = self._get_vrfid_from_net(net)
+        response = self.driver.get_vrf_details(
+            n_context.get_admin_context(), **kwargs)
+        self.assertEqual(['100.100.100.0/24', '50.50.50.0/24'],
+            response['vrf_subnets'])
+
+    def test_no_nat_cidrs_private_net_router(self):
+        # test no nat cidrs extension for private network connected to a
+        # router with out external gateway
+        host = 'host1'
+        kwargs = {'apic:no_nat_cidrs': ['10.10.10.0/24']}
+        net = self._make_network(
+            self.fmt, 'net10', True,
+            arg_list=tuple(list(kwargs.keys())), **kwargs)
+        net_id = net['network']['id']
+
+        subnet_id = self._make_subnet(
+            self.fmt, net, '10.0.1.1', '10.0.1.0/24')['subnet']['id']
+
+        rtr = self._make_router(
+            self.fmt, net['network']['tenant_id'], 'router1')['router']
+        self._router_interface_action('add', rtr['id'], subnet_id, None)
+
+        port = self._make_port(self.fmt, net_id)['port']
+        port_id = port['id']
+        port = self._bind_port_to_host(port_id, host)['port']
+        self.assertEqual('ovs', port['binding:vif_type'])
+
+        # Call the RPC handler.
+        request = {
+            'device': 'tap' + port_id,
+            'timestamp': 12345,
+            'request_id': 'a_request'
+        }
+        response = self.driver.request_endpoint_details(
+            n_context.get_admin_context(), request=request, host=host)
+        self.assertEqual(set(['10.0.1.0/24', '10.10.10.0/24']),
+            set(response['gbp_details']['vrf_subnets']))
+        net = self._show('networks', net['network']['id'])
+        kwargs = self._get_vrfid_from_net(net)
+        response = self.driver.get_vrf_details(
+            n_context.get_admin_context(), **kwargs)
+        self.assertEqual(set(['10.0.1.0/24', '10.10.10.0/24']),
+            set(response['vrf_subnets']))
+
+    def test_no_nat_cidrs_private_net_router_with_gw_case1(self):
+        # test no nat cidrs extension for private network connected to a
+        # router with an external gateway.
+        # extension available in private network, but not in external network
+        ext_net = self._make_ext_network('ext-net1',
+                                         dn=self.dn_t1_l1_n1)
+        self._make_subnet(
+            self.fmt, {'network': ext_net}, '100.100.100.1',
+            '100.100.100.0/24')
+
+        host = 'host1'
+        kwargs = {'apic:no_nat_cidrs': ['10.10.10.0/24']}
+        net = self._make_network(
+            self.fmt, 'net10', True,
+            arg_list=tuple(list(kwargs.keys())), **kwargs)
+        net_id = net['network']['id']
+
+        subnet_id = self._make_subnet(
+            self.fmt, net, '10.0.1.1', '10.0.1.0/24')['subnet']['id']
+
+        rtr = self._make_router(
+            self.fmt, net['network']['tenant_id'], 'router1',
+            external_gateway_info={'network_id': ext_net['id']})['router']
+        self._router_interface_action('add', rtr['id'], subnet_id, None)
+
+        port = self._make_port(self.fmt, net_id)['port']
+        port_id = port['id']
+        port = self._bind_port_to_host(port_id, host)['port']
+        self.assertEqual('ovs', port['binding:vif_type'])
+
+        # Call the RPC handler.
+        request = {
+            'device': 'tap' + port_id,
+            'timestamp': 12345,
+            'request_id': 'a_request'
+        }
+        response = self.driver.request_endpoint_details(
+            n_context.get_admin_context(), request=request, host=host)
+        self.assertEqual(['10.0.1.0/24', '10.10.10.0/24'],
+            response['gbp_details']['vrf_subnets'])
+        net = self._show('networks', net['network']['id'])
+        kwargs = self._get_vrfid_from_net(net)
+        response = self.driver.get_vrf_details(
+            n_context.get_admin_context(), **kwargs)
+        self.assertEqual(['10.0.1.0/24', '10.10.10.0/24'],
+            response['vrf_subnets'])
+
+    def test_no_nat_cidrs_private_net_router_with_gw_case2(self):
+        # test no nat cidrs extension for private network connected to a
+        # router with an external gateway.
+        # extension available in external network, but not in private network
+        ext_net = self._make_ext_network('ext-net1',
+                                         dn=self.dn_t1_l1_n1)
+        ext_net = self._update(
+            'networks', ext_net['id'],
+            {'network':
+             {'apic:no_nat_cidrs': ['50.50.50.0/24']}})
+        self._make_subnet(
+            self.fmt, {'network': ext_net['network']}, '100.100.100.1',
+            '100.100.100.0/24')
+
+        host = 'host1'
+        net = self._make_network(
+            self.fmt, 'net10', True)
+        net_id = net['network']['id']
+
+        subnet_id = self._make_subnet(
+            self.fmt, net, '10.0.1.1', '10.0.1.0/24')['subnet']['id']
+
+        rtr = self._make_router(
+            self.fmt, net['network']['tenant_id'], 'router1',
+            external_gateway_info={'network_id': ext_net['network']['id']})
+        self._router_interface_action('add', rtr['router']['id'],
+                                    subnet_id, None)
+
+        port = self._make_port(self.fmt, net_id)['port']
+        port_id = port['id']
+        port = self._bind_port_to_host(port_id, host)['port']
+        self.assertEqual('ovs', port['binding:vif_type'])
+
+        # Call the RPC handler.
+        request = {
+            'device': 'tap' + port_id,
+            'timestamp': 12345,
+            'request_id': 'a_request'
+        }
+        response = self.driver.request_endpoint_details(
+            n_context.get_admin_context(), request=request, host=host)
+        self.assertEqual(['10.0.1.0/24', '50.50.50.0/24'],
+            response['gbp_details']['vrf_subnets'])
+        net = self._show('networks', net['network']['id'])
+        kwargs = self._get_vrfid_from_net(net)
+        response = self.driver.get_vrf_details(
+            n_context.get_admin_context(), **kwargs)
+        self.assertEqual(['10.0.1.0/24', '50.50.50.0/24'],
+            response['vrf_subnets'])
+
+    def test_no_nat_cidrs_private_net_router_with_gw_case3(self):
+        # test no nat cidrs extension for private network connected to a
+        # router with an external gateway.
+        # extension available in both external network and private network
+        ext_net = self._make_ext_network('ext-net1',
+                                         dn=self.dn_t1_l1_n1)
+        ext_net = self._update(
+            'networks', ext_net['id'],
+            {'network':
+             {'apic:no_nat_cidrs': ['50.50.50.0/24']}})
+        self._make_subnet(
+            self.fmt, {'network': ext_net['network']}, '100.100.100.1',
+            '100.100.100.0/24')
+
+        host = 'host1'
+        kwargs = {'apic:no_nat_cidrs': ['10.10.10.0/24']}
+        net = self._make_network(
+            self.fmt, 'net10', True,
+            arg_list=tuple(list(kwargs.keys())), **kwargs)
+        net_id = net['network']['id']
+
+        subnet_id = self._make_subnet(
+            self.fmt, net, '10.0.1.1', '10.0.1.0/24')['subnet']['id']
+
+        rtr = self._make_router(
+            self.fmt, net['network']['tenant_id'], 'router1',
+            external_gateway_info={'network_id': ext_net['network']['id']})
+        self._router_interface_action('add', rtr['router']['id'],
+            subnet_id, None)
+
+        port = self._make_port(self.fmt, net_id)['port']
+        port_id = port['id']
+        port = self._bind_port_to_host(port_id, host)['port']
+        self.assertEqual('ovs', port['binding:vif_type'])
+
+        # Call the RPC handler.
+        request = {
+            'device': 'tap' + port_id,
+            'timestamp': 12345,
+            'request_id': 'a_request'
+        }
+        response = self.driver.request_endpoint_details(
+            n_context.get_admin_context(), request=request, host=host)
+        self.assertEqual(
+            set(['10.0.1.0/24', '50.50.50.0/24', '10.10.10.0/24']),
+            set(response['gbp_details']['vrf_subnets']))
+        net = self._show('networks', net['network']['id'])
+        kwargs = self._get_vrfid_from_net(net)
+        response = self.driver.get_vrf_details(
+            n_context.get_admin_context(), **kwargs)
+        self.assertEqual(
+            set(['10.0.1.0/24', '50.50.50.0/24', '10.10.10.0/24']),
+            set(response['vrf_subnets']))
+
+    def test_no_nat_cidrs_with_address_scope(self):
+        # test no nat cidrs extension when address scope available
+        # Create address scope.
+        scope = self._make_address_scope(
+            self.fmt, 4, name='as1')['address_scope']
+        scope_id = scope['id']
+
+        # Create subnet pool.
+        pool = self._make_subnetpool(self.fmt, ['10.0.0.0/8'], name='sp1',
+                                     tenant_id=self._tenant_id,
+                                     address_scope_id=scope_id,
+                                     default_prefixlen=24)['subnetpool']
+        pool_id = pool['id']
+
+        # Create network.
+        kwargs = {'apic:no_nat_cidrs': ['10.10.10.0/24']}
+        net = self._make_network(
+            self.fmt, 'net10', True,
+            arg_list=tuple(list(kwargs.keys())), **kwargs)
+        net_id = net['network']['id']
+
+        # Create subnet1.
+        subnet = self._make_subnet(
+            self.fmt, net, '10.0.1.1', '10.0.1.0/24',
+            subnetpool_id=pool_id)['subnet']
+        subnet_id = subnet['id']
+
+        # Create an external network and make it as a gateway to router
+        ext_net = self._make_ext_network('ext-net1',
+                                         dn=self.dn_t1_l1_n1)
+        ext_net = self._update(
+            'networks', ext_net['id'],
+            {'network':
+             {'apic:no_nat_cidrs': ['50.50.50.0/24']}})
+        self._make_subnet(
+            self.fmt, {'network': ext_net['network']}, '100.100.100.1',
+            '100.100.100.0/24')
+
+        rtr = self._make_router(
+            self.fmt, self._tenant_id, 'router1',
+            external_gateway_info={'network_id': ext_net['network']['id']})
+        self._router_interface_action('add', rtr['router']['id'],
+                                    subnet_id, None)
+
+        host = 'host1'
+        port = self._make_port(self.fmt, net_id)['port']
+        port_id = port['id']
+        port = self._bind_port_to_host(port_id, host)['port']
+        self.assertEqual('ovs', port['binding:vif_type'])
+
+        # Call the RPC handler.
+        request = {
+            'device': 'tap' + port_id,
+            'timestamp': 12345,
+            'request_id': 'a_request'
+        }
+        response = self.driver.request_endpoint_details(
+            n_context.get_admin_context(), request=request, host=host)
+        self.assertEqual(
+            set(['10.0.0.0/8', '50.50.50.0/24', '10.10.10.0/24']),
+            set(response['gbp_details']['vrf_subnets']))
+        net = self._show('networks', net['network']['id'])
+        kwargs = self._get_vrfid_from_net(net)
+        response = self.driver.get_vrf_details(
+            n_context.get_admin_context(), **kwargs)
+        self.assertEqual(
+            set(['10.0.0.0/8', '50.50.50.0/24', '10.10.10.0/24']),
+            set(response['vrf_subnets']))