VPNaaS: Provide Endpoint groups capability.

This commit implements the new tables for VPN endpoint-groups, and the
CRUD REST API. Validation logic has been added for the reference
implementation. Migration files are created, so that the needed tables
are created. Unit tests have been added for the API and database.

The neutron client and Horizon support for this new API will be handled
separately, as will API test cases for the new API.

This new API is a prerequisite for providing the multiple local subnet
feature, which will use this API and make changes to existing APIs. At
that time, migration and backward compatibility support will be provided,
so that users can migrate smoothly to the new API (since there is no
support for micro-versioning).

APIImpact

Depends-On: Ia729bd0c6967fa2b8c698495aa360f340b42d98a
Change-Id: I6e10590a988312eafca076a14be38b19e2d44a87
Partial-Bug: 1459423

neutron_vpnaas/db/migration/alembic_migrations/versions/liberty/expand/41b509d10b5e_vpnaas_endpoint_groups.py

@@ -0,0 +1,59 @@
+#    (c) Copyright 2015 Cisco Systems Inc.
+#    All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""VPNaaS endpoint groups
+
+Revision ID: 41b509d10b5e
+Revises: 24f28869838b
+Create Date: 2015-08-06 18:21:03.241664
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '41b509d10b5e'
+down_revision = '24f28869838b'
+
+from alembic import op
+import sqlalchemy as sa
+
+from neutron.api.v2 import attributes as attr
+
+from neutron_vpnaas.services.vpn.common import constants
+
+
+def upgrade():
+    op.create_table(
+        'vpn_endpoint_groups',
+        sa.Column('id', sa.String(length=36), nullable=False,
+                  primary_key=True),
+        sa.Column('tenant_id', sa.String(length=attr.TENANT_ID_MAX_LEN),
+                  index=True),
+        sa.Column('name', sa.String(length=attr.NAME_MAX_LEN)),
+        sa.Column('description', sa.String(length=attr.DESCRIPTION_MAX_LEN)),
+        sa.Column('endpoint_type',
+                  sa.Enum(constants.SUBNET_ENDPOINT, constants.CIDR_ENDPOINT,
+                          constants.VLAN_ENDPOINT, constants.NETWORK_ENDPOINT,
+                          constants.ROUTER_ENDPOINT,
+                          name='endpoint_type'),
+                  nullable=False),
+    )
+    op.create_table(
+        'vpn_endpoints',
+        sa.Column('endpoint', sa.String(length=255), nullable=False),
+        sa.Column('endpoint_group_id', sa.String(36), nullable=False),
+        sa.ForeignKeyConstraint(['endpoint_group_id'],
+                                ['vpn_endpoint_groups.id'],
+                                ondelete='CASCADE'),
+        sa.PrimaryKeyConstraint('endpoint', 'endpoint_group_id'),
+    )

neutron_vpnaas/db/vpn/vpn_db.py

@@ -72,6 +72,9 @@ class VPNPluginDb(vpnaas.VPNPluginBase, base_db.CommonDbMixin):
                     raise vpnaas.IPsecPolicyNotFound(ipsecpolicy_id=v_id)
                 elif issubclass(model, vpn_models.VPNService):
                     raise vpnaas.VPNServiceNotFound(vpnservice_id=v_id)
+                elif issubclass(model, vpn_models.VPNEndpointGroup):
+                    raise vpnaas.VPNEndpointGroupNotFound(
+                        endpoint_group_id=v_id)
                 ctx.reraise = True
         return r
 
@@ -515,6 +518,64 @@ class VPNPluginDb(vpnaas.VPNPluginBase, base_db.CommonDbMixin):
                     subnet_id=subnet_id,
                     vpnservice_id=vpnservices['id'])
 
+    def _make_endpoint_group_dict(self, endpoint_group, fields=None):
+        res = {'id': endpoint_group['id'],
+               'tenant_id': endpoint_group['tenant_id'],
+               'name': endpoint_group['name'],
+               'description': endpoint_group['description'],
+               'type': endpoint_group['endpoint_type'],
+               'endpoints': [ep['endpoint']
+                             for ep in endpoint_group['endpoints']]}
+        return self._fields(res, fields)
+
+    def create_endpoint_group(self, context, endpoint_group):
+        group = endpoint_group['endpoint_group']
+        tenant_id = self._get_tenant_id_for_create(context, group)
+        validator = self._get_validator()
+        with context.session.begin(subtransactions=True):
+            validator.validate_endpoint_group(context, group)
+            endpoint_group_db = vpn_models.VPNEndpointGroup(
+                id=uuidutils.generate_uuid(),
+                tenant_id=tenant_id,
+                name=group['name'],
+                description=group['description'],
+                endpoint_type=group['type'])
+            context.session.add(endpoint_group_db)
+            for endpoint in group['endpoints']:
+                endpoint_db = vpn_models.VPNEndpoint(
+                    endpoint=endpoint,
+                    endpoint_group_id=endpoint_group_db['id']
+                )
+                context.session.add(endpoint_db)
+        return self._make_endpoint_group_dict(endpoint_group_db)
+
+    def update_endpoint_group(self, context, endpoint_group_id,
+                              endpoint_group):
+        group_changes = endpoint_group['endpoint_group']
+        # Note: Endpoints cannot be changed, so will not do validation
+        with context.session.begin(subtransactions=True):
+            endpoint_group_db = self._get_resource(context,
+                                                   vpn_models.VPNEndpointGroup,
+                                                   endpoint_group_id)
+            endpoint_group_db.update(group_changes)
+        return self._make_endpoint_group_dict(endpoint_group_db)
+
+    def delete_endpoint_group(self, context, endpoint_group_id):
+        with context.session.begin(subtransactions=True):
+            endpoint_group_db = self._get_resource(
+                context, vpn_models.VPNEndpointGroup, endpoint_group_id)
+            context.session.delete(endpoint_group_db)
+
+    def get_endpoint_group(self, context, endpoint_group_id, fields=None):
+        endpoint_group_db = self._get_resource(
+            context, vpn_models.VPNEndpointGroup, endpoint_group_id)
+        return self._make_endpoint_group_dict(endpoint_group_db, fields)
+
+    def get_endpoint_groups(self, context, filters=None, fields=None):
+        return self._get_collection(context, vpn_models.VPNEndpointGroup,
+                                    self._make_endpoint_group_dict,
+                                    filters=filters, fields=fields)
+
 
 class VPNPluginRpcDbMixin(object):
     def _get_agent_hosting_vpn_services(self, context, host):

neutron_vpnaas/db/vpn/vpn_models.py

@@ -13,6 +13,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from neutron.api.v2 import attributes as attr
 from neutron.db import l3_db
 from neutron.db import model_base
 from neutron.db import models_v2
@@ -20,6 +21,8 @@ from neutron.db import models_v2
 import sqlalchemy as sa
 from sqlalchemy import orm
 
+from neutron_vpnaas.services.vpn.common import constants
+
 
 class IPsecPeerCidr(model_base.BASEV2):
     """Internal representation of a IPsec Peer Cidrs."""
@@ -35,8 +38,8 @@ class IPsecPeerCidr(model_base.BASEV2):
 class IPsecPolicy(model_base.BASEV2, models_v2.HasId, models_v2.HasTenant):
     """Represents a v2 IPsecPolicy Object."""
     __tablename__ = 'ipsecpolicies'
-    name = sa.Column(sa.String(255))
-    description = sa.Column(sa.String(255))
+    name = sa.Column(sa.String(attr.NAME_MAX_LEN))
+    description = sa.Column(sa.String(attr.DESCRIPTION_MAX_LEN))
     transform_protocol = sa.Column(sa.Enum("esp", "ah", "ah-esp",
                                            name="ipsec_transform_protocols"),
                                    nullable=False)
@@ -61,8 +64,8 @@ class IPsecPolicy(model_base.BASEV2, models_v2.HasId, models_v2.HasTenant):
 class IKEPolicy(model_base.BASEV2, models_v2.HasId, models_v2.HasTenant):
     """Represents a v2 IKEPolicy Object."""
     __tablename__ = 'ikepolicies'
-    name = sa.Column(sa.String(255))
-    description = sa.Column(sa.String(255))
+    name = sa.Column(sa.String(attr.NAME_MAX_LEN))
+    description = sa.Column(sa.String(attr.DESCRIPTION_MAX_LEN))
     auth_algorithm = sa.Column(sa.Enum("sha1",
                                        name="vpn_auth_algorithms"),
                                nullable=False)
@@ -87,8 +90,8 @@ class IPsecSiteConnection(model_base.BASEV2,
                           models_v2.HasId, models_v2.HasTenant):
     """Represents a IPsecSiteConnection Object."""
     __tablename__ = 'ipsec_site_connections'
-    name = sa.Column(sa.String(255))
-    description = sa.Column(sa.String(255))
+    name = sa.Column(sa.String(attr.NAME_MAX_LEN))
+    description = sa.Column(sa.String(attr.DESCRIPTION_MAX_LEN))
     peer_address = sa.Column(sa.String(255), nullable=False)
     peer_id = sa.Column(sa.String(255), nullable=False)
     route_mode = sa.Column(sa.String(8), nullable=False)
@@ -125,8 +128,8 @@ class IPsecSiteConnection(model_base.BASEV2,
 
 class VPNService(model_base.BASEV2, models_v2.HasId, models_v2.HasTenant):
     """Represents a v2 VPNService Object."""
-    name = sa.Column(sa.String(255))
-    description = sa.Column(sa.String(255))
+    name = sa.Column(sa.String(attr.NAME_MAX_LEN))
+    description = sa.Column(sa.String(attr.DESCRIPTION_MAX_LEN))
     status = sa.Column(sa.String(16), nullable=False)
     admin_state_up = sa.Column(sa.Boolean(), nullable=False)
     external_v4_ip = sa.Column(sa.String(16))
@@ -141,3 +144,33 @@ class VPNService(model_base.BASEV2, models_v2.HasId, models_v2.HasTenant):
         IPsecSiteConnection,
         backref='vpnservice',
         cascade="all, delete-orphan")
+
+
+class VPNEndpoint(model_base.BASEV2):
+    """Endpoints used in VPN connections.
+
+    All endpoints in a group must be of the same type. Note: the endpoint
+    is an 'opaque' field used to hold different endpoint types, and be
+    flexible enough to use for future types.
+    """
+    __tablename__ = 'vpn_endpoints'
+    endpoint = sa.Column(sa.String(255), nullable=False, primary_key=True)
+    endpoint_group_id = sa.Column(sa.String(36),
+                                  sa.ForeignKey('vpn_endpoint_groups.id',
+                                                ondelete="CASCADE"),
+                                  primary_key=True)
+
+
+class VPNEndpointGroup(model_base.BASEV2, models_v2.HasId,
+                       models_v2.HasTenant):
+    """Collection of endpoints of a specific type, for VPN connections."""
+    __tablename__ = 'vpn_endpoint_groups'
+    name = sa.Column(sa.String(attr.NAME_MAX_LEN))
+    description = sa.Column(sa.String(attr.DESCRIPTION_MAX_LEN))
+    endpoint_type = sa.Column(sa.Enum(*constants.VPN_SUPPORTED_ENDPOINT_TYPES,
+                                      name="vpn_endpoint_type"),
+                              nullable=False)
+    endpoints = orm.relationship(VPNEndpoint,
+                                 backref='endpoint_group',
+                                 lazy='joined',
+                                 cascade='all, delete, delete-orphan')

neutron_vpnaas/db/vpn/vpn_validator.py

@@ -16,11 +16,16 @@ import netaddr
 import socket
 
 from neutron.api.v2 import attributes
+from neutron.common import exceptions as nexception
 from neutron.db import l3_db
 from neutron import manager
-from neutron.plugins.common import constants
+from neutron.plugins.common import constants as nconstants
+from oslo_log import log as logging
 
 from neutron_vpnaas.extensions import vpnaas
+from neutron_vpnaas.services.vpn.common import constants
+
+LOG = logging.getLogger(__name__)
 
 
 class VpnReferenceValidator(object):
@@ -35,7 +40,7 @@ class VpnReferenceValidator(object):
             return self._l3_plugin
         except AttributeError:
             self._l3_plugin = manager.NeutronManager.get_service_plugins().get(
-                constants.L3_ROUTER_NAT)
+                nconstants.L3_ROUTER_NAT)
             return self._l3_plugin
 
     @property
@@ -143,3 +148,42 @@ class VpnReferenceValidator(object):
         for IPSec Policy validation.
         """
         pass
+
+    def _validate_cidrs(self, cidrs):
+        """Ensure valid IPv4/6 CIDRs."""
+        for cidr in cidrs:
+            msg = attributes._validate_subnet(cidr)
+            if msg:
+                raise vpnaas.InvalidEndpointInEndpointGroup(
+                    group_type=constants.CIDR_ENDPOINT, endpoint=cidr,
+                    why=_("Invalid CIDR"))
+
+    def _validate_subnets(self, context, subnet_ids):
+        """Ensure UUIDs OK and subnets exist."""
+        for subnet_id in subnet_ids:
+            msg = attributes._validate_uuid(subnet_id)
+            if msg:
+                raise vpnaas.InvalidEndpointInEndpointGroup(
+                    group_type=constants.SUBNET_ENDPOINT, endpoint=subnet_id,
+                    why=_('Invalid UUID'))
+            try:
+                self.core_plugin.get_subnet(context, subnet_id)
+            except nexception.SubnetNotFound:
+                raise vpnaas.NonExistingSubnetInEndpointGroup(
+                    subnet=subnet_id)
+
+    def validate_endpoint_group(self, context, endpoint_group):
+        """Reference validator for endpoint group.
+
+        Ensures that there is at least one endpoint, all the endpoints in the
+        group are of the same type, and that the endpoints are "valid".
+        Note: Only called for create, as endpoints cannot be changed.
+        """
+        endpoints = endpoint_group['endpoints']
+        if not endpoints:
+            raise vpnaas.MissingEndpointForEndpointGroup(group=endpoint_group)
+        group_type = endpoint_group['type']
+        if group_type == constants.CIDR_ENDPOINT:
+            self._validate_cidrs(endpoints)
+        elif group_type == constants.SUBNET_ENDPOINT:
+            self._validate_subnets(context, endpoints)

neutron_vpnaas/extensions/vpnaas.py

@@ -21,9 +21,11 @@ from neutron.api import extensions
 from neutron.api.v2 import attributes as attr
 from neutron.api.v2 import resource_helper
 from neutron.common import exceptions as nexception
-from neutron.plugins.common import constants
+from neutron.plugins.common import constants as nconstants
 from neutron.services import service_base
 
+from neutron_vpnaas.services.vpn.common import constants
+
 
 class VPNServiceNotFound(nexception.NotFound):
     message = _("VPNService %(vpnservice_id)s could not be found")
@@ -96,6 +98,23 @@ class ExternalNetworkHasNoSubnet(nexception.BadRequest):
                 "no %(ip_version)s subnet")
 
 
+class VPNEndpointGroupNotFound(nexception.NotFound):
+    message = _("Endpoint group %(endpoint_group_id)s could not be found")
+
+
+class InvalidEndpointInEndpointGroup(nexception.InvalidInput):
+    message = _("Endpoint '%(endpoint)s' is invalid for group "
+                "type '%(group_type)s': %(why)s")
+
+
+class MissingEndpointForEndpointGroup(nexception.BadRequest):
+    message = _("No endpoints specified for endpoint group '%(group)s'")
+
+
+class NonExistingSubnetInEndpointGroup(nexception.InvalidInput):
+    message = _("Subnet %(subnet)s in endpoint group does not exist")
+
+
 vpn_supported_initiators = ['bi-directional', 'response-only']
 vpn_supported_encryption_algorithms = ['3des', 'aes-128',
                                        'aes-192', 'aes-256']
@@ -343,7 +362,32 @@ RESOURCE_ATTRIBUTE_MAP = {
                 'default': 'group5',
                 'validate': {'type:values': vpn_supported_pfs},
                 'is_visible': True}
-    }
+    },
+
+    'endpoint_groups': {
+        'id': {'allow_post': False, 'allow_put': False,
+               'validate': {'type:uuid': None},
+               'is_visible': True,
+               'primary_key': True},
+        'tenant_id': {'allow_post': True, 'allow_put': False,
+                      'validate': {'type:string': attr.TENANT_ID_MAX_LEN},
+                      'required_by_policy': True,
+                      'is_visible': True},
+        'name': {'allow_post': True, 'allow_put': True,
+                 'validate': {'type:string': attr.NAME_MAX_LEN},
+                 'is_visible': True, 'default': ''},
+        'description': {'allow_post': True, 'allow_put': True,
+                        'validate': {'type:string': attr.DESCRIPTION_MAX_LEN},
+                        'is_visible': True, 'default': ''},
+        'type': {'allow_post': True, 'allow_put': False,
+                 'validate': {
+                     'type:values': constants.VPN_SUPPORTED_ENDPOINT_TYPES,
+                 },
+                 'is_visible': True},
+        'endpoints': {'allow_post': True, 'allow_put': False,
+                      'convert_to': attr.convert_to_list,
+                      'is_visible': True},
+    },
 }
 
 
@@ -379,7 +423,7 @@ class Vpnaas(extensions.ExtensionDescriptor):
         attr.PLURALS.update(plural_mappings)
         return resource_helper.build_resource_info(plural_mappings,
                                                    RESOURCE_ATTRIBUTE_MAP,
-                                                   constants.VPN,
+                                                   nconstants.VPN,
                                                    register_quota=True,
                                                    translate_name=True)
 
@@ -402,10 +446,10 @@ class Vpnaas(extensions.ExtensionDescriptor):
 class VPNPluginBase(service_base.ServicePluginBase):
 
     def get_plugin_name(self):
-        return constants.VPN
+        return nconstants.VPN
 
     def get_plugin_type(self):
-        return constants.VPN
+        return nconstants.VPN
 
     def get_plugin_description(self):
         return 'VPN service plugin'
@@ -491,3 +535,24 @@ class VPNPluginBase(service_base.ServicePluginBase):
     @abc.abstractmethod
     def delete_ipsecpolicy(self, context, ipsecpolicy_id):
         pass
+
+    @abc.abstractmethod
+    def create_endpoint_group(self, context, endpoint_group):
+        pass
+
+    @abc.abstractmethod
+    def update_endpoint_group(self, context, endpoint_group_id,
+                              endpoint_group):
+        pass
+
+    @abc.abstractmethod
+    def delete_endpoint_group(self, context, endpoint_group_id):
+        pass
+
+    @abc.abstractmethod
+    def get_endpoint_group(self, context, endpoint_group_id, fields=None):
+        pass
+
+    @abc.abstractmethod
+    def get_endpoint_groups(self, context, filters=None, fields=None):
+        pass

neutron_vpnaas/services/vpn/common/constants.py

@@ -0,0 +1,32 @@
+# Copyright 2015 Cisco Systems, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+# Endpoint group types
+SUBNET_ENDPOINT = 'subnet'
+CIDR_ENDPOINT = 'cidr'
+VLAN_ENDPOINT = 'vlan'
+NETWORK_ENDPOINT = 'network'
+ROUTER_ENDPOINT = 'router'
+
+# NOTE: Type usage...
+# IPSec local endpoints - subnet, IPSec peer endpoints - cidr
+# BGP VPN local endpoints - network
+# Direct connect style endpoints - vlan
+# IMPORTANT: The ordering of these is important, as it is used in an enum
+# for the database (and migration script). Only add to this list.
+VPN_SUPPORTED_ENDPOINT_TYPES = [
+    SUBNET_ENDPOINT, CIDR_ENDPOINT, NETWORK_ENDPOINT,
+    VLAN_ENDPOINT, ROUTER_ENDPOINT,
+]

neutron_vpnaas/tests/unit/db/vpn/test_vpn_db.py

@@ -28,16 +28,18 @@ from neutron.db import servicetype_db as sdb
 from neutron import extensions as nextensions
 from neutron.extensions import l3 as l3_exception
 from neutron import manager
-from neutron.plugins.common import constants
+from neutron.plugins.common import constants as nconstants
 from neutron.scheduler import l3_agent_scheduler
 from neutron.tests.unit.db import test_db_base_plugin_v2 as test_db_plugin
 from neutron.tests.unit.extensions import test_l3 as test_l3_plugin
+from oslo_db import exception as db_exc
 from oslo_utils import uuidutils
 import six
 import webob.exc
 
 from neutron_vpnaas.db.vpn import vpn_db
 from neutron_vpnaas.db.vpn import vpn_models
+from neutron_vpnaas.services.vpn.common import constants
 from neutron_vpnaas.services.vpn import plugin as vpn_plugin
 from neutron_vpnaas.tests import base
 
@@ -402,14 +404,14 @@ class VPNTestMixin(object):
 
     def _set_active(self, model, resource_id):
         service_plugin = manager.NeutronManager.get_service_plugins()[
-            constants.VPN]
+            nconstants.VPN]
         adminContext = context.get_admin_context()
         with adminContext.session.begin(subtransactions=True):
             resource_db = service_plugin._get_resource(
                 adminContext,
                 model,
                 resource_id)
-            resource_db.status = constants.ACTIVE
+            resource_db.status = nconstants.ACTIVE
 
 
 class VPNPluginDbTestCase(VPNTestMixin,
@@ -419,7 +421,7 @@ class VPNPluginDbTestCase(VPNTestMixin,
               vpnaas_provider=None):
         if not vpnaas_provider:
             vpnaas_provider = (
-                constants.VPN +
+                nconstants.VPN +
                 ':vpnaas:neutron_vpnaas.services.vpn.'
                 'service_drivers.ipsec.IPsecVPNDriver:default')
         bits = vpnaas_provider.split(':')
@@ -450,8 +452,8 @@ class VPNPluginDbTestCase(VPNTestMixin,
         self.plugin = vpn_plugin.VPNPlugin()
         ext_mgr = api_extensions.PluginAwareExtensionManager(
             extensions_path,
-            {constants.CORE: self.core_plugin,
-             constants.VPN: self.plugin}
+            {nconstants.CORE: self.core_plugin,
+             nconstants.VPN: self.plugin}
         )
         app = config.load_paste_app('extensions_test_app')
         self.ext_api = api_extensions.ExtensionMiddleware(app, ext_mgr=ext_mgr)
@@ -1726,7 +1728,8 @@ class TestVpnDatabase(base.NeutronDbPluginV2TestCase, NeutronResourcesMixin):
         # Get the plugins
         self.core_plugin = manager.NeutronManager.get_plugin()
         self.l3_plugin = manager.NeutronManager.get_service_plugins().get(
-            constants.L3_ROUTER_NAT)
+            nconstants.L3_ROUTER_NAT)
+
         # Create VPN database instance
         self.plugin = vpn_db.VPNPluginDb()
         self.tenant_id = uuidutils.generate_uuid()
@@ -1771,3 +1774,167 @@ class TestVpnDatabase(base.NeutronDbPluginV2TestCase, NeutronResourcesMixin):
                                                           v4_ip=external_v4_ip,
                                                           v6_ip=external_v6_ip)
         self.assertDictSupersetOf(expected, mod_service)
+
+    def prepare_endpoint_info(self, group_type, endpoints):
+        return {'endpoint_group': {'tenant_id': self.tenant_id,
+                                   'name': 'my endpoint group',
+                                   'description': 'my description',
+                                   'type': group_type,
+                                   'endpoints': endpoints}}
+
+    def test_endpoint_group_create_and_with_cidrs(self):
+        """Verify create endpoint group using CIDRs."""
+        info = self.prepare_endpoint_info(constants.CIDR_ENDPOINT,
+                                          ['10.10.10.0/24', '20.20.20.0/24'])
+        expected = info['endpoint_group']
+        new_endpoint_group = self.plugin.create_endpoint_group(self.context,
+                                                               info)
+        self.assertDictSupersetOf(expected, new_endpoint_group)
+
+    def test_endpoint_group_create_with_subnets(self):
+        """Verify create endpoint group using subnets."""
+        # Skip validation for subnets, as validation is checked in other tests
+        mock.patch.object(self.l3_plugin, "get_subnet").start()
+        private_subnet, router = self.create_basic_topology()
+        private_net2 = self.create_network(overrides={'name': 'private2'})
+        overrides = {'name': 'private-subnet2',
+                     'cidr': '10.1.0.0/24',
+                     'gateway_ip': '10.1.0.1',
+                     'network_id': private_net2['id']}
+        private_subnet2 = self.create_subnet(overrides=overrides)
+        self.create_router_port_for_subnet(router, private_subnet2)
+
+        info = self.prepare_endpoint_info(constants.SUBNET_ENDPOINT,
+                                          [private_subnet['id'],
+                                           private_subnet2['id']])
+        expected = info['endpoint_group']
+        new_endpoint_group = self.plugin.create_endpoint_group(self.context,
+                                                               info)
+        self.assertDictSupersetOf(expected, new_endpoint_group)
+
+    def test_endpoint_group_create_with_vlans(self):
+        """Verify endpoint group using VLANs."""
+        info = self.prepare_endpoint_info(constants.VLAN_ENDPOINT,
+                                          ['100', '200', '300'])
+        expected = info['endpoint_group']
+        new_endpoint_group = self.plugin.create_endpoint_group(self.context,
+                                                               info)
+        self.assertDictSupersetOf(expected, new_endpoint_group)
+
+    def helper_create_endpoint_group(self, info):
+        """Helper to create endpoint group database entry."""
+        try:
+            actual = self.plugin.create_endpoint_group(self.context, info)
+        except db_exc.DBError as e:
+            self.fail("Endpoint create in prep for test failed: %s" % e)
+        self.assertDictSupersetOf(info['endpoint_group'], actual)
+        self.assertIn('id', actual)
+        return actual['id']
+
+    def check_endpoint_group_entry(self, endpoint_group_id, expected_info,
+                                   should_exist=True):
+        try:
+            endpoint_group = self.plugin.get_endpoint_group(self.context,
+                                                            endpoint_group_id)
+            is_found = True
+        except vpnaas.VPNEndpointGroupNotFound:
+            is_found = False
+        except Exception as e:
+            self.fail("Unexpected exception getting endpoint group: %s" % e)
+
+        if should_exist != is_found:
+            self.fail("Endpoint group should%(expected)s exist, but "
+                      "did%(actual)s exist" %
+                      {'expected': '' if should_exist else ' not',
+                       'actual': '' if is_found else ' not'})
+        if is_found:
+            self.assertDictSupersetOf(expected_info, endpoint_group)
+
+    def test_delete_endpoint_group(self):
+        """Test that endpoint group is deleted."""
+        info = self.prepare_endpoint_info(constants.CIDR_ENDPOINT,
+                                          ['10.10.10.0/24', '20.20.20.0/24'])
+        expected = info['endpoint_group']
+        group_id = self.helper_create_endpoint_group(info)
+        self.check_endpoint_group_entry(group_id, expected, should_exist=True)
+
+        self.plugin.delete_endpoint_group(self.context, group_id)
+        self.check_endpoint_group_entry(group_id, expected, should_exist=False)
+
+        self.assertRaises(vpnaas.VPNEndpointGroupNotFound,
+                          self.plugin.delete_endpoint_group,
+                          self.context, group_id)
+
+    def test_show_endpoint_group(self):
+        """Test showing a single endpoint group."""
+        info = self.prepare_endpoint_info(constants.CIDR_ENDPOINT,
+                                          ['10.10.10.0/24', '20.20.20.0/24'])
+        expected = info['endpoint_group']
+        group_id = self.helper_create_endpoint_group(info)
+        self.check_endpoint_group_entry(group_id, expected, should_exist=True)
+
+        actual = self.plugin.get_endpoint_group(self.context, group_id)
+        self.assertDictSupersetOf(expected, actual)
+
+    def test_fail_showing_non_existent_endpoint_group(self):
+        """Test failure to show non-existent endpoint gourp."""
+        self.assertRaises(vpnaas.VPNEndpointGroupNotFound,
+                          self.plugin.get_endpoint_group,
+                          self.context, uuidutils.generate_uuid())
+
+    def test_list_endpoint_groups(self):
+        """Test listing multiple endpoint groups."""
+        # Skip validation for subnets, as validation is checked in other tests
+        mock.patch.object(self.l3_plugin, "get_subnet").start()
+        info1 = self.prepare_endpoint_info(constants.CIDR_ENDPOINT,
+                                           ['10.10.10.0/24', '20.20.20.0/24'])
+        expected1 = info1['endpoint_group']
+        group_id1 = self.helper_create_endpoint_group(info1)
+        self.check_endpoint_group_entry(group_id1, expected1,
+                                        should_exist=True)
+
+        info2 = self.prepare_endpoint_info(constants.SUBNET_ENDPOINT,
+                                           [uuidutils.generate_uuid(),
+                                            uuidutils.generate_uuid()])
+        expected2 = info2['endpoint_group']
+        group_id2 = self.helper_create_endpoint_group(info2)
+        self.check_endpoint_group_entry(group_id2, expected2,
+                                        should_exist=True)
+        expected1.update({'id': group_id1})
+        expected2.update({'id': group_id2})
+        # Note: Subnet IDs could be in any order - force ascending
+        expected2['endpoints'].sort()
+        expected = [expected1, expected2]
+        actual = self.plugin.get_endpoint_groups(self.context,
+            fields=('type', 'tenant_id', 'endpoints',
+                    'name', 'description', 'id'))
+        self.assertEqual(expected, actual)
+
+    def test_update_endpoint_group(self):
+        """Test updating endpoint group information."""
+        info = self.prepare_endpoint_info(constants.CIDR_ENDPOINT,
+                                          ['10.10.10.0/24', '20.20.20.0/24'])
+        expected = info['endpoint_group']
+        group_id = self.helper_create_endpoint_group(info)
+        self.check_endpoint_group_entry(group_id, expected, should_exist=True)
+
+        group_updates = {'endpoint_group': {'name': 'new name',
+                                            'description': 'new description'}}
+        updated_group = self.plugin.update_endpoint_group(self.context,
+                                                          group_id,
+                                                          group_updates)
+
+        # Check what was returned, and what is stored in database
+        self.assertDictSupersetOf(group_updates['endpoint_group'],
+                                  updated_group)
+        expected.update(group_updates['endpoint_group'])
+        self.check_endpoint_group_entry(group_id, expected,
+                                        should_exist=True)
+
+    def test_fail_updating_non_existent_group(self):
+        """Test fail updating a non-existent group."""
+        group_updates = {'endpoint_group': {'name': 'new name'}}
+        self.assertRaises(
+            vpnaas.VPNEndpointGroupNotFound,
+            self.plugin.update_endpoint_group,
+            self.context, uuidutils.generate_uuid(), group_updates)

neutron_vpnaas/tests/unit/extensions/test_vpnaas.py

@@ -15,12 +15,13 @@
 
 import copy
 import mock
-from neutron.plugins.common import constants
+from neutron.plugins.common import constants as nconstants
 from neutron.tests.unit.api.v2 import test_base as test_api_v2
 from oslo_utils import uuidutils
 from webob import exc
 
 from neutron_vpnaas.extensions import vpnaas
+from neutron_vpnaas.services.vpn.common import constants
 from neutron_vpnaas.tests import base
 
 _uuid = uuidutils.generate_uuid
@@ -34,9 +35,10 @@ class VpnaasExtensionTestCase(base.ExtensionTestCase):
         super(VpnaasExtensionTestCase, self).setUp()
         plural_mappings = {'ipsecpolicy': 'ipsecpolicies',
                            'ikepolicy': 'ikepolicies',
-                           'ipsec_site_connection': 'ipsec-site-connections'}
+                           'ipsec_site_connection': 'ipsec-site-connections',
+                           'endpoint_group': 'endpoint-groups'}
         self._setUpExtension(
-            'neutron_vpnaas.extensions.vpnaas.VPNPluginBase', constants.VPN,
+            'neutron_vpnaas.extensions.vpnaas.VPNPluginBase', nconstants.VPN,
             vpnaas.RESOURCE_ATTRIBUTE_MAP, vpnaas.Vpnaas,
             'vpn', plural_mappings=plural_mappings,
             use_quota=True)
@@ -518,3 +520,146 @@ class VpnaasExtensionTestCase(base.ExtensionTestCase):
     def test_ipsec_site_connection_delete(self):
         """Test case to delete a ipsec_site_connection."""
         self._test_entity_delete('ipsec_site_connection')
+
+    def helper_test_endpoint_group_create(self, data):
+        """Check that the endpoint_group_create works.
+
+        Uses passed in endpoint group information, which specifies an
+        endpoint type and values.
+        """
+        data['endpoint_group'].update({'tenant_id': _uuid(),
+                                       'name': 'my endpoint group',
+                                       'description': 'my description'})
+        return_value = copy.copy(data['endpoint_group'])
+        return_value.update({'id': _uuid()})
+
+        instance = self.plugin.return_value
+        instance.create_endpoint_group.return_value = return_value
+        res = self.api.post(_get_path('vpn/endpoint-groups', fmt=self.fmt),
+                            self.serialize(data),
+                            content_type='application/%s' % self.fmt)
+        instance.create_endpoint_group.assert_called_with(
+            mock.ANY, endpoint_group=data)
+        self.assertEqual(exc.HTTPCreated.code, res.status_int)
+        res = self.deserialize(res)
+        self.assertIn('endpoint_group', res)
+        self.assertEqual(res['endpoint_group'], return_value)
+
+    def test_create_cidr_endpoint_group_create(self):
+        """Test creation of CIDR type endpoint group."""
+        data = {'endpoint_group':
+                {'type': constants.CIDR_ENDPOINT,
+                 'endpoints': ['10.10.10.0/24', '20.20.20.0/24']}}
+        self.helper_test_endpoint_group_create(data)
+
+    def test_create_subnet_endpoint_group_create(self):
+        """Test creation of subnet type endpoint group."""
+        data = {'endpoint_group':
+                {'type': constants.SUBNET_ENDPOINT,
+                 'endpoints': [_uuid(), _uuid()]}}
+        self.helper_test_endpoint_group_create(data)
+
+    def test_create_vlan_endpoint_group_create(self):
+        """Test creation of VLAN type endpoint group."""
+        data = {'endpoint_group':
+                {'type': constants.VLAN_ENDPOINT,
+                 'endpoints': ['100', '200', '300', '400']}}
+        self.helper_test_endpoint_group_create(data)
+
+    def test_get_endpoint_group(self):
+        """Test show for endpoint group."""
+        endpoint_group_id = _uuid()
+        return_value = {'id': endpoint_group_id,
+                        'tenant_id': _uuid(),
+                        'name': 'my-endpoint-group',
+                        'description': 'my endpoint group',
+                        'type': constants.CIDR_ENDPOINT,
+                        'endpoints': ['10.10.10.0/24']}
+
+        instance = self.plugin.return_value
+        instance.get_endpoint_group.return_value = return_value
+
+        res = self.api.get(_get_path('vpn/endpoint-groups',
+                                     id=endpoint_group_id,
+                                     fmt=self.fmt))
+
+        instance.get_endpoint_group.assert_called_with(mock.ANY,
+                                                       endpoint_group_id,
+                                                       fields=mock.ANY)
+        self.assertEqual(res.status_int, exc.HTTPOk.code)
+        res = self.deserialize(res)
+        self.assertIn('endpoint_group', res)
+        self.assertEqual(res['endpoint_group'], return_value)
+
+    def test_endpoint_group_list(self):
+        """Test listing all endpoint groups."""
+        return_value = [{'id': _uuid(),
+                         'tenant_id': _uuid(),
+                         'name': 'my-endpoint-group',
+                         'description': 'my endpoint group',
+                         'type': constants.CIDR_ENDPOINT,
+                         'endpoints': ['10.10.10.0/24']},
+                        {'id': _uuid(),
+                         'tenant_id': _uuid(),
+                         'name': 'another-endpoint-group',
+                         'description': 'second endpoint group',
+                         'type': constants.VLAN_ENDPOINT,
+                         'endpoints': ['100', '200', '300']}]
+
+        instance = self.plugin.return_value
+        instance.get_endpoint_groups.return_value = return_value
+
+        res = self.api.get(_get_path('vpn/endpoint-groups', fmt=self.fmt))
+
+        instance.get_endpoint_groups.assert_called_with(mock.ANY,
+                                                        fields=mock.ANY,
+                                                        filters=mock.ANY)
+        self.assertEqual(res.status_int, exc.HTTPOk.code)
+
+    def test_endpoint_group_delete(self):
+        """Test deleting an endpoint group."""
+        self._test_entity_delete('endpoint_group')
+
+    def test_endpoint_group_update(self):
+        """Test updating endpoint_group."""
+        endpoint_group_id = _uuid()
+        update_data = {'endpoint_group': {'description': 'new description'}}
+        return_value = {'id': endpoint_group_id,
+                        'tenant_id': _uuid(),
+                        'name': 'my-endpoint-group',
+                        'description': 'new_description',
+                        'type': constants.CIDR_ENDPOINT,
+                        'endpoints': ['10.10.10.0/24']}
+
+        instance = self.plugin.return_value
+        instance.update_endpoint_group.return_value = return_value
+
+        res = self.api.put(_get_path('vpn/endpoint-groups',
+                                     id=endpoint_group_id,
+                                     fmt=self.fmt),
+                           self.serialize(update_data))
+
+        instance.update_endpoint_group.assert_called_with(
+            mock.ANY, endpoint_group_id, endpoint_group=update_data)
+        self.assertEqual(res.status_int, exc.HTTPOk.code)
+        res = self.deserialize(res)
+        self.assertIn('endpoint_group', res)
+        self.assertEqual(res['endpoint_group'], return_value)
+
+    def test_fail_updating_endpoints_in_endpoint_group(self):
+        """Test fails to update the endpoints in an endpoint group.
+
+        This documents that we are not allowing endpoints to be updated
+        (currently), as doing so, implies that the connection using the
+        enclosing endpoint group would also need to be updated. For now,
+        a new endpoint group can be created, and the connection can be
+        updated to point to the new endpoint group.
+        """
+        endpoint_group_id = _uuid()
+        update_data = {'endpoint_group': {'endpoints': ['10.10.10.0/24']}}
+        res = self.api.put(_get_path('vpn/endpoint-groups',
+                                     id=endpoint_group_id,
+                                     fmt=self.fmt),
+                           params=self.serialize(update_data),
+                           expect_errors=True)
+        self.assertEqual(exc.HTTPBadRequest.code, res.status_int)

neutron_vpnaas/tests/unit/services/vpn/service_drivers/test_ipsec.py

@@ -16,14 +16,16 @@
 import mock
 import socket
 
+from neutron.common import exceptions as nexception
 from neutron import context as n_ctx
 from neutron.db import l3_db
 from neutron.db import servicetype_db as st_db
-from neutron.plugins.common import constants
+from neutron.plugins.common import constants as nconstants
 from oslo_config import cfg
 from oslo_utils import uuidutils
 
 from neutron_vpnaas.extensions import vpnaas
+from neutron_vpnaas.services.vpn.common import constants
 from neutron_vpnaas.services.vpn import plugin as vpn_plugin
 from neutron_vpnaas.services.vpn.service_drivers import ipsec as ipsec_driver
 from neutron_vpnaas.services.vpn.service_drivers \
@@ -58,13 +60,13 @@ class TestValidatorSelection(base.BaseTestCase):
         # TODO(armax): remove this if branch as soon as the ServiceTypeManager
         # API for adding provider configurations becomes available
         if not hasattr(st_db.ServiceTypeManager, 'add_provider_configuration'):
-            vpnaas_provider = (constants.VPN + ':vpnaas:' +
+            vpnaas_provider = (nconstants.VPN + ':vpnaas:' +
                                IPSEC_SERVICE_DRIVER + ':default')
             cfg.CONF.set_override(
                 'service_provider', [vpnaas_provider], 'service_providers')
         else:
             vpnaas_provider = [{
-                'service_type': constants.VPN,
+                'service_type': nconstants.VPN,
                 'name': 'vpnaas',
                 'driver': IPSEC_SERVICE_DRIVER,
                 'default': True
@@ -92,7 +94,7 @@ class TestIPsecDriverValidation(base.BaseTestCase):
         self.l3_plugin = mock.Mock()
         mock.patch(
             'neutron.manager.NeutronManager.get_service_plugins',
-            return_value={constants.L3_ROUTER_NAT: self.l3_plugin}).start()
+            return_value={nconstants.L3_ROUTER_NAT: self.l3_plugin}).start()
         self.core_plugin = mock.Mock()
         mock.patch('neutron.manager.NeutronManager.get_plugin',
                    return_value=self.core_plugin).start()
@@ -306,6 +308,67 @@ class TestIPsecDriverValidation(base.BaseTestCase):
                 self.validator.validate_ipsec_site_connection,
                 self.context, ipsec_sitecon, version)
 
+    def test_endpoints_all_cidrs_in_endpoint_group(self):
+        """All endpoints in the endpoint group are valid CIDRs."""
+        endpoint_group = {'type': constants.CIDR_ENDPOINT,
+                          'endpoints': ['10.10.10.0/24', '20.20.20.0/24']}
+        try:
+            self.validator.validate_endpoint_group(self.context,
+                                                   endpoint_group)
+        except Exception:
+            self.fail("All CIDRs in endpoint_group should be valid")
+
+    def test_endpoints_all_subnets_in_endpoint_group(self):
+        """All endpoints in the endpoint group are valid subnets."""
+        endpoint_group = {'type': constants.SUBNET_ENDPOINT,
+                          'endpoints': [_uuid(), _uuid()]}
+        try:
+            self.validator.validate_endpoint_group(self.context,
+                                                   endpoint_group)
+        except Exception:
+            self.fail("All subnets in endpoint_group should be valid")
+
+    def test_mixed_endpoint_types_in_endpoint_group(self):
+        """Fail when mixing types of endpoints in endpoint group."""
+        endpoint_group = {'type': constants.CIDR_ENDPOINT,
+                          'endpoints': ['10.10.10.0/24', _uuid()]}
+        self.assertRaises(vpnaas.InvalidEndpointInEndpointGroup,
+                          self.validator.validate_endpoint_group,
+                          self.context, endpoint_group)
+        endpoint_group = {'type': constants.SUBNET_ENDPOINT,
+                          'endpoints': [_uuid(), '10.10.10.0/24']}
+        self.assertRaises(vpnaas.InvalidEndpointInEndpointGroup,
+                          self.validator.validate_endpoint_group,
+                          self.context, endpoint_group)
+
+    def test_missing_endpoints_for_endpoint_group(self):
+        endpoint_group = {'type': constants.CIDR_ENDPOINT,
+                          'endpoints': []}
+        self.assertRaises(vpnaas.MissingEndpointForEndpointGroup,
+                          self.validator.validate_endpoint_group,
+                          self.context, endpoint_group)
+
+    def test_fail_bad_cidr_in_endpoint_group(self):
+        """Testing catches bad CIDR.
+
+        Just check one case, as CIDR validator used has good test coverage.
+        """
+        endpoint_group = {'type': constants.CIDR_ENDPOINT,
+                          'endpoints': ['10.10.10.10/24', '20.20.20.1']}
+        self.assertRaises(vpnaas.InvalidEndpointInEndpointGroup,
+                          self.validator.validate_endpoint_group,
+                          self.context, endpoint_group)
+
+    def test_unknown_subnet_in_endpoint_group(self):
+        subnet_id = _uuid()
+        self.core_plugin.get_subnet.side_effect = nexception.SubnetNotFound(
+            subnet_id=subnet_id)
+        endpoint_group = {'type': constants.SUBNET_ENDPOINT,
+                          'endpoints': [subnet_id]}
+        self.assertRaises(vpnaas.NonExistingSubnetInEndpointGroup,
+                          self.validator.validate_endpoint_group,
+                          self.context, endpoint_group)
+
 
 class FakeSqlQueryObject(dict):
     """To fake SqlAlchemy query object and access keys as attributes."""
@@ -330,7 +393,7 @@ class TestIPsecDriver(base.BaseTestCase):
         service_plugin_p = mock.patch(
             'neutron.manager.NeutronManager.get_service_plugins')
         get_service_plugin = service_plugin_p.start()
-        get_service_plugin.return_value = {constants.L3_ROUTER_NAT: plugin}
+        get_service_plugin.return_value = {nconstants.L3_ROUTER_NAT: plugin}
         self.svc_plugin = mock.Mock()
         self.svc_plugin.get_l3_agents_hosting_routers.return_value = [l3_agent]
         self._fake_vpn_router_id = _uuid()