Convert EndpointMap to not require per-service VIP parameters

Currently we have a hard-coded set of per-service parameters, which
will cause problems for custom roles and full composability.

As a first step towards making this more configurable, remove the
hard-coded per-service parameters from overcloud.yaml, and adjust
the EndpointMap generation to instead accept two mappings, the
ServiceNetMap and a mapping of networks to IPs (effectively this
just moves the map lookup inside the endpoint map instead of
inside overcloud.yaml)

Change-Id: Ib522e89c36eed2115a6586dd5a6770907d9b33db
Partially-Implements: blueprint custom-roles

environments/enable-tls.yaml

@@ -38,7 +38,6 @@ parameter_defaults:
     ManilaInternal: {protocol: 'http', port: '8786', host: 'IP_ADDRESS'}
     ManilaPublic: {protocol: 'https', port: '13786', host: 'CLOUDNAME'}
     MysqlInternal: {protocol: 'mysql+pymysql', port: '3306', host: 'IP_ADDRESS'}
-    MysqlNoBracketsInternal: {protocol: 'mysql+pymysql', port: '3306', host: 'IP_ADDRESS'}
     NeutronAdmin: {protocol: 'http', port: '9696', host: 'IP_ADDRESS'}
     NeutronInternal: {protocol: 'http', port: '9696', host: 'IP_ADDRESS'}
     NeutronPublic: {protocol: 'https', port: '13696', host: 'CLOUDNAME'}

network/endpoints/build_endpoint_map.py

@@ -30,7 +30,9 @@ import yaml
 (IN_FILE, OUT_FILE) = ('endpoint_data.yaml', 'endpoint_map.yaml')
 
 SUBST = (SUBST_IP_ADDRESS, SUBST_CLOUDNAME) = ('IP_ADDRESS', 'CLOUDNAME')
-PARAMS = (PARAM_CLOUDNAME, PARAM_ENDPOINTMAP) = ('CloudName', 'EndpointMap')
+PARAMS = (PARAM_CLOUDNAME, PARAM_ENDPOINTMAP, PARAM_NETIPMAP,
+          PARAM_SERVICENETMAP) = (
+          'CloudName', 'EndpointMap', 'NetIpMap', 'ServiceNetMap')
 FIELDS = (F_PORT, F_PROTOCOL, F_HOST) = ('port', 'protocol', 'host')
 
 ENDPOINT_TYPES = frozenset(['Internal', 'Public', 'Admin'])
@@ -56,16 +58,8 @@ def load_endpoint_data(infile=None):
         return yaml.safe_load(f)
 
 
-def vip_param_name(endpoint_type_defn):
+def net_param_name(endpoint_type_defn):
-    return endpoint_type_defn['vip_param'] + 'VirtualIP'
+    return endpoint_type_defn['net_param'] + 'Network'
-
-
-def vip_param_names(config):
-    def ep_types(svc):
-        return (v for k, v in svc.items() if k in ENDPOINT_TYPES or not k)
-
-    return set(vip_param_name(defn)
-               for svc in config.values() for defn in ep_types(svc))
 
 
 def endpoint_map_default(config):
@@ -91,9 +85,9 @@ def make_parameter(ptype, default, description=None):
 
 
 def template_parameters(config):
-    params = collections.OrderedDict((n, make_parameter('string', ''))
+    params = collections.OrderedDict()
-                                     for n in sorted(vip_param_names(config)))
+    params[PARAM_NETIPMAP] = make_parameter('json', {}, 'The Net IP map')
-
+    params[PARAM_SERVICENETMAP] = make_parameter('json', {}, 'The Service Net map')
     params[PARAM_ENDPOINTMAP] = make_parameter('json',
                                                endpoint_map_default(config),
                                                'Mapping of service endpoint '
@@ -111,7 +105,7 @@ def template_parameters(config):
 def template_output_definition(endpoint_name,
                                endpoint_variant,
                                endpoint_type,
-                               vip_param,
+                               net_param,
                                uri_suffix=None,
                                name_override=None):
     def extract_field(field):
@@ -122,11 +116,29 @@ def template_output_definition(endpoint_name,
 
     port = extract_field(F_PORT)
     protocol = extract_field(F_PROTOCOL)
+    host_nobrackets = {
+        'str_replace': collections.OrderedDict([
+            ('template', extract_field(F_HOST)),
+            ('params', {
+                SUBST_IP_ADDRESS: {'get_param':
+                                   ['NetIpMap',
+                                    {'get_param': ['ServiceNetMap',
+                                     net_param]}]},
+                SUBST_CLOUDNAME: {'get_param': PARAM_CLOUDNAME},
+            })
+        ])
+    }
     host = {
         'str_replace': collections.OrderedDict([
             ('template', extract_field(F_HOST)),
             ('params', {
-                SUBST_IP_ADDRESS: {'get_param': vip_param},
+                SUBST_IP_ADDRESS: {'get_param':
+                                   ['NetIpMap',
+                                    {'str_replace':
+                                    {'template': 'NETWORK_uri',
+                                     'params': {'NETWORK':
+                                     {'get_param': ['ServiceNetMap',
+                                                    net_param]}}}}]},
                 SUBST_CLOUDNAME: {'get_param': PARAM_CLOUDNAME},
             })
         ])
@@ -140,6 +152,7 @@ def template_output_definition(endpoint_name,
                                                             endpoint_type)
 
     return name, {
+        'host_nobrackets': host_nobrackets,
         'host': host,
         'port': extract_field('port'),
         'protocol': extract_field('protocol'),
@@ -160,10 +173,9 @@ def template_endpoint_items(config):
                                             {'': None}).items():
                 name_override = defn.get('names', {}).get(variant)
                 yield template_output_definition(ep_name, variant, ep_type,
-                                                 vip_param_name(defn),
+                                                 net_param_name(defn),
                                                  suffix,
                                                  name_override)
-
     return itertools.chain.from_iterable(sorted(get_svc_endpoints(ep_name,
                                                                   svc))
                                          for (ep_name,

network/endpoints/endpoint_data.yaml

@@ -3,46 +3,46 @@
 
 Aodh:
     Internal:
-        vip_param: AodhApi
+        net_param: AodhApi
     Public:
-        vip_param: Public
+        net_param: Public
     Admin:
-        vip_param: AodhApi
+        net_param: AodhApi
     port: 8042
 
 Ceilometer:
     Internal:
-        vip_param: CeilometerApi
+        net_param: CeilometerApi
     Public:
-        vip_param: Public
+        net_param: Public
     Admin:
-        vip_param: CeilometerApi
+        net_param: CeilometerApi
     port: 8777
 
 Gnocchi:
     Internal:
-        vip_param: GnocchiApi
+        net_param: GnocchiApi
     Public:
-        vip_param: Public
+        net_param: Public
     Admin:
-        vip_param: GnocchiApi
+        net_param: GnocchiApi
     port: 8041
 
 Cinder:
     Internal:
-        vip_param: CinderApi
+        net_param: CinderApi
         uri_suffixes:
             '': /v1/%(tenant_id)s
             V2: /v2/%(tenant_id)s
             V3: /v3/%(tenant_id)s
     Public:
-        vip_param: Public
+        net_param: Public
         uri_suffixes:
             '': /v1/%(tenant_id)s
             V2: /v2/%(tenant_id)s
             V3: /v3/%(tenant_id)s
     Admin:
-        vip_param: CinderApi
+        net_param: CinderApi
         uri_suffixes:
             '': /v1/%(tenant_id)s
             V2: /v2/%(tenant_id)s
@@ -51,63 +51,57 @@ Cinder:
 
 Glance:
     Internal:
-        vip_param: GlanceApi
+        net_param: GlanceApi
     Public:
-        vip_param: Public
+        net_param: Public
     Admin:
-        vip_param: GlanceApi
+        net_param: GlanceApi
     port: 9292
 
 GlanceRegistry:
     Internal:
-        vip_param: GlanceRegistry
+        net_param: GlanceRegistry
     port: 9191
 
 Mysql:
     Internal:
-        vip_param: Mysql
+        net_param: Mysql
-    protocol: mysql+pymysql
-    port: 3306
-
-MysqlNoBrackets:
-    Internal:
-        vip_param: MysqlNoBrackets
     protocol: mysql+pymysql
     port: 3306
 
 Heat:
     Internal:
-        vip_param: HeatApi
+        net_param: HeatApi
         uri_suffixes:
             '': /v1/%(tenant_id)s
     Public:
-        vip_param: Public
+        net_param: Public
         uri_suffixes:
             '': /v1/%(tenant_id)s
     Admin:
-        vip_param: HeatApi
+        net_param: HeatApi
         uri_suffixes:
             '': /v1/%(tenant_id)s
     port: 8004
 
 HeatCfn:
     Internal:
-        vip_param: HeatApi
+        net_param: HeatApi
         uri_suffixes:
             '': /v1
     Public:
-        vip_param: Public
+        net_param: Public
         uri_suffixes:
             '': /v1
     Admin:
-        vip_param: HeatApi
+        net_param: HeatApi
         uri_suffixes:
             '': /v1
     port: 8000
 
 Horizon:
     Public:
-        vip_param: Public
+        net_param: Public
         uri_suffixes:
             '': /dashboard
     port: 80
@@ -116,7 +110,7 @@ Horizon:
 # Required for https://bugs.launchpad.net/puppet-nova/+bug/1542486
 Keystone:
     Internal:
-        vip_param: KeystonePublicApi
+        net_param: KeystonePublicApi
         uri_suffixes:
             '': /v2.0
             EC2: /v2.0/ec2tokens
@@ -124,12 +118,12 @@ Keystone:
         names:
             EC2: KeystoneEC2
     Public:
-        vip_param: Public
+        net_param: Public
         uri_suffixes:
             '': /v2.0
             V3: /v3
     Admin:
-        vip_param: KeystoneAdminApi
+        net_param: KeystoneAdminApi
         uri_suffixes:
             '': /v2.0
             V3: /v3
@@ -138,17 +132,17 @@ Keystone:
 
 Manila:
     Internal:
-        vip_param: ManilaApi
+        net_param: ManilaApi
         uri_suffixes:
             '': /v2/%(tenant_id)s
             V1: /v1/%(tenant_id)s
     Public:
-        vip_param: Public
+        net_param: Public
         uri_suffixes:
             '': /v2/%(tenant_id)s
             V1: /v1/%(tenant_id)s
     Admin:
-        vip_param: ManilaApi
+        net_param: ManilaApi
         uri_suffixes:
             '': /v2/%(tenant_id)s
             V1: /v1/%(tenant_id)s
@@ -156,50 +150,50 @@ Manila:
 
 Neutron:
     Internal:
-        vip_param: NeutronApi
+        net_param: NeutronApi
     Public:
-        vip_param: Public
+        net_param: Public
     Admin:
-        vip_param: NeutronApi
+        net_param: NeutronApi
     port: 9696
 
 Nova:
     Internal:
-        vip_param: NovaApi
+        net_param: NovaApi
         uri_suffixes:
             '': /v2.1
     Public:
-        vip_param: Public
+        net_param: Public
         uri_suffixes:
             '': /v2.1
     Admin:
-        vip_param: NovaApi
+        net_param: NovaApi
         uri_suffixes:
             '': /v2.1
     port: 8774
 
 NovaVNCProxy:
     Internal:
-        vip_param: NovaApi
+        net_param: NovaApi
     Public:
-        vip_param: Public
+        net_param: Public
     Admin:
-        vip_param: NovaApi
+        net_param: NovaApi
     port: 6080
 
 Swift:
     Internal:
-        vip_param: SwiftProxy
+        net_param: SwiftProxy
         uri_suffixes:
             '': /v1/AUTH_%(tenant_id)s
             S3:
     Public:
-        vip_param: Public
+        net_param: Public
         uri_suffixes:
             '': /v1/AUTH_%(tenant_id)s
             S3:
     Admin:
-        vip_param: SwiftProxy
+        net_param: SwiftProxy
         uri_suffixes:
             '':
             S3:
@@ -207,30 +201,30 @@ Swift:
 
 Sahara:
     Internal:
-        vip_param: SaharaApi
+        net_param: SaharaApi
         uri_suffixes:
             '': /v1.1/%(tenant_id)s
     Public:
-        vip_param: Public
+        net_param: Public
         uri_suffixes:
             '': /v1.1/%(tenant_id)s
     Admin:
-        vip_param: SaharaApi
+        net_param: SaharaApi
         uri_suffixes:
             '': /v1.1/%(tenant_id)s
     port: 8386
 
 Ironic:
     Internal:
-        vip_param: IronicApi
+        net_param: IronicApi
         uri_suffixes:
             '': /v1
     Public:
-        vip_param: Public
+        net_param: Public
         uri_suffixes:
             '': /v1
     Admin:
-        vip_param: IronicApi
+        net_param: IronicApi
         uri_suffixes:
             '': /v1
     port: 6385

overcloud.yaml

@@ -141,6 +141,7 @@ parameters:
       BlockStorageHostnameResolveNetwork: internal_api
       ObjectStorageHostnameResolveNetwork: internal_api
       CephStorageHostnameResolveNetwork: storage
+      PublicNetwork: external
     description: Mapping of service_name -> network name. Typically set
                  via parameter_defaults in the resource registry.
     type: json
@@ -396,24 +397,8 @@ resources:
     type: OS::TripleO::EndpointMap
     properties:
       CloudName: {get_param: CloudName}
-      CeilometerApiVirtualIP: {get_attr: [VipMap, net_ip_uri_map, {get_param: [ServiceNetMap, CeilometerApiNetwork]}]}
+      NetIpMap: {get_attr: [VipMap, net_ip_map]}
-      AodhApiVirtualIP: {get_attr: [VipMap, net_ip_uri_map, {get_param: [ServiceNetMap, AodhApiNetwork]}]}
+      ServiceNetMap: {get_param: ServiceNetMap}
-      CinderApiVirtualIP: {get_attr: [VipMap, net_ip_uri_map, {get_param: [ServiceNetMap, CinderApiNetwork]}]}
-      GlanceApiVirtualIP: {get_attr: [VipMap, net_ip_uri_map, {get_param: [ServiceNetMap, GlanceApiNetwork]}]}
-      GlanceRegistryVirtualIP: {get_attr: [VipMap, net_ip_uri_map, {get_param: [ServiceNetMap, GlanceRegistryNetwork]}]}
-      GnocchiApiVirtualIP: {get_attr: [VipMap, net_ip_uri_map, {get_param: [ServiceNetMap, GnocchiApiNetwork]}]}
-      HeatApiVirtualIP: {get_attr: [VipMap, net_ip_uri_map, {get_param: [ServiceNetMap, HeatApiNetwork]}]}
-      IronicApiVirtualIP: {get_attr: [VipMap, net_ip_uri_map, {get_param: [ServiceNetMap, IronicApiNetwork]}]}
-      KeystoneAdminApiVirtualIP: {get_attr: [VipMap, net_ip_uri_map, {get_param: [ServiceNetMap, KeystoneAdminApiNetwork]}]}
-      KeystonePublicApiVirtualIP: {get_attr: [VipMap, net_ip_uri_map, {get_param: [ServiceNetMap, KeystonePublicApiNetwork]}]}
-      ManilaApiVirtualIP: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, ManilaApiNetwork]}]}
-      MysqlVirtualIP: {get_attr: [VipMap, net_ip_uri_map, {get_param: [ServiceNetMap, MysqlNetwork]}]}
-      MysqlNoBracketsVirtualIP: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, MysqlNetwork]}]}
-      NeutronApiVirtualIP: {get_attr: [VipMap, net_ip_uri_map, {get_param: [ServiceNetMap, NeutronApiNetwork]}]}
-      NovaApiVirtualIP: {get_attr: [VipMap, net_ip_uri_map, {get_param: [ServiceNetMap, NovaApiNetwork]}]}
-      SaharaApiVirtualIP: {get_attr: [VipMap, net_ip_uri_map, {get_param: [ServiceNetMap, SaharaApiNetwork]}]}
-      SwiftProxyVirtualIP: {get_attr: [VipMap, net_ip_uri_map, {get_param: [ServiceNetMap, SwiftProxyNetwork]}]}
-      PublicVirtualIP: {get_attr: [VipMap, net_ip_uri_map, external]}
 
   ControllerServiceChain:
     type: OS::TripleO::Services

puppet/services/aodh-base.yaml

@@ -85,7 +85,7 @@ outputs:
         aodh::keystone::auth::tenant: 'service'
         aodh::db::mysql::user: aodh
         aodh::db::mysql::password: {get_param: AodhPassword}
-        aodh::db::mysql::host: {get_param: [EndpointMap, MysqlNoBracketsInternal, host]}
+        aodh::db::mysql::host: {get_param: [EndpointMap, MysqlInternal, host_nobrackets]}
         aodh::db::mysql::dbname: aodh
         aodh::db::mysql::allowed_hosts:
           - '%'

puppet/services/ceilometer-base.yaml

@@ -103,7 +103,7 @@ outputs:
         ceilometer::rabbit_use_ssl: {get_param: RabbitClientUseSSL}
         ceilometer::rabbit_port: {get_param: RabbitClientPort}
         ceilometer::db::mysql::user: ceilometer
-        ceilometer::db::mysql::host: {get_param: [EndpointMap, MysqlNoBracketsInternal, host]}
+        ceilometer::db::mysql::host: {get_param: [EndpointMap, MysqlInternal, host_nobrackets]}
         ceilometer::db::mysql::dbname: ceilometer
         ceilometer::db::mysql::allowed_hosts:
           - '%'

puppet/services/cinder-base.yaml

@@ -58,7 +58,7 @@ outputs:
         cinder::rabbit_password: {get_param: RabbitPassword}
         cinder::rabbit_port: {get_param: RabbitClientPort}
         cinder::db::mysql::user: cinder
-        cinder::db::mysql::host: {get_param: [EndpointMap, MysqlNoBracketsInternal, host]}
+        cinder::db::mysql::host: {get_param: [EndpointMap, MysqlInternal, host_nobrackets]}
         cinder::db::mysql::dbname: cinder
         cinder::db::mysql::allowed_hosts:
           - '%'

puppet/services/glance-registry.yaml

@@ -45,7 +45,7 @@ outputs:
         glance::registry::debug: {get_param: Debug}
         glance::registry::workers: {get_param: GlanceWorkers}
         glance::db::mysql::user: glance
-        glance::db::mysql::host: {get_param: [EndpointMap, MysqlNoBracketsInternal, host]}
+        glance::db::mysql::host: {get_param: [EndpointMap, MysqlInternal, host_nobrackets]}
         glance::db::mysql::dbname: glance
         glance::db::mysql::allowed_hosts:
           - '%'

puppet/services/gnocchi-base.yaml

@@ -89,7 +89,7 @@ outputs:
         gnocchi::statsd::flush_delay: 10
         gnocchi::statsd::archive_policy_name: 'low'
         gnocchi::db::mysql::user: gnocchi
-        gnocchi::db::mysql::host: {get_param: [EndpointMap, MysqlNoBracketsInternal, host]}
+        gnocchi::db::mysql::host: {get_param: [EndpointMap, MysqlInternal, host_nobrackets]}
         gnocchi::db::mysql::dbname: gnocchi
         gnocchi::db::mysql::allowed_hosts:
           - '%'

puppet/services/heat-engine.yaml

@@ -58,7 +58,7 @@ outputs:
             heat::db::mysql::password: {get_param: HeatPassword}
             heat::keystone::domain::domain_password: {get_param: HeatStackDomainAdminPassword}
             heat::db::mysql::user: heat
-            heat::db::mysql::host: {get_param: [EndpointMap, MysqlNoBracketsInternal, host]}
+            heat::db::mysql::host: {get_param: [EndpointMap, MysqlInternal, host_nobrackets]}
             heat::db::mysql::dbname: heat
             heat::db::mysql::allowed_hosts:
               - '%'

puppet/services/ironic-base.yaml

@@ -59,7 +59,7 @@ outputs:
         ironic::rabbit_use_ssl: {get_param: RabbitClientUseSSL}
         ironic::db::mysql::password: {get_param: IronicPassword}
         ironic::db::mysql::user: ironic
-        ironic::db::mysql::host: {get_param: [EndpointMap, MysqlNoBracketsInternal, host]}
+        ironic::db::mysql::host: {get_param: [EndpointMap, MysqlInternal, host_nobrackets]}
         ironic::db::mysql::dbname: ironic
         ironic::db::mysql::allowed_hosts:
           - '%'

puppet/services/keystone.yaml

@@ -112,7 +112,7 @@ outputs:
         keystone_enable_db_purge: {get_param: KeystoneEnableDBPurge}
         keystone::public_endpoint: {get_param: [EndpointMap, KeystonePublic, uri_no_suffix]}
         keystone::db::mysql::user: keystone
-        keystone::db::mysql::host: {get_param: [EndpointMap, MysqlNoBracketsInternal, host]}
+        keystone::db::mysql::host: {get_param: [EndpointMap, MysqlInternal, host_nobrackets]}
         keystone::db::mysql::dbname: keystone
         keystone::db::mysql::allowed_hosts:
           - '%'

puppet/services/manila-base.yaml

@@ -93,7 +93,7 @@ outputs:
         manila::rabbit_port: {get_param: RabbitClientPort}
         manila::debug: {get_param: Debug}
         manila::db::mysql::user: manila
-        manila::db::mysql::host: {get_param: [EndpointMap, MysqlNoBracketsInternal, host]}
+        manila::db::mysql::host: {get_param: [EndpointMap, MysqlInternal, host_nobrackets]}
         manila::db::mysql::dbname: manila
         manila::db::database_db_max_retries: -1
         manila::db::database_max_retries: -1

puppet/services/neutron-server.yaml

@@ -82,7 +82,7 @@ outputs:
             neutron::server::sync_db: true
             neutron::db::mysql::password: {get_param: NeutronPassword}
             neutron::db::mysql::user: neutron
-            neutron::db::mysql::host: {get_param: [EndpointMap, MysqlNoBracketsInternal, host]}
+            neutron::db::mysql::host: {get_param: [EndpointMap, MysqlInternal, host_nobrackets]}
             neutron::db::mysql::dbname: ovs_neutron
             neutron::db::mysql::allowed_hosts:
               - '%'

puppet/services/nova-base.yaml

@@ -70,14 +70,14 @@ outputs:
               - '/nova_api'
         nova::db::mysql::password: {get_input: nova_password}
         nova::db::mysql::user: nova
-        nova::db::mysql::host: {get_param: [EndpointMap, MysqlNoBracketsInternal, host]}
+        nova::db::mysql::host: {get_param: [EndpointMap, MysqlInternal, host_nobrackets]}
         nova::db::mysql::dbname: nova
         nova::db::mysql::allowed_hosts:
           - '%'
           - "%{hiera('mysql_bind_host')}"
         nova::db::mysql_api::password: {get_input: nova_password}
         nova::db::mysql_api::user: nova_api
-        nova::db::mysql_api::host: {get_param: [EndpointMap, MysqlNoBracketsInternal, host]}
+        nova::db::mysql_api::host: {get_param: [EndpointMap, MysqlInternal, host_nobrackets]}
         nova::db::mysql_api::dbname: nova_api
         nova::db::mysql_api::allowed_hosts:
           - '%'
@@ -96,13 +96,13 @@ outputs:
         nova::notification_driver: messagingv2
         nova::network::neutron::neutron_auth_type: 'v3password'
         nova::db::mysql::user: nova
-        nova::db::mysql::host: {get_param: [EndpointMap, MysqlNoBracketsInternal, host]}
+        nova::db::mysql::host: {get_param: [EndpointMap, MysqlInternal, host_nobrackets]}
         nova::db::mysql::dbname: nova
         nova::db::mysql::allowed_hosts:
           - '%'
           - "%{hiera('mysql_bind_host')}"
         nova::db::mysql_api::user: nova_api
-        nova::db::mysql_api::host: {get_param: [EndpointMap, MysqlNoBracketsInternal, host]}
+        nova::db::mysql_api::host: {get_param: [EndpointMap, MysqlInternal, host_nobrackets]}
         nova::db::mysql_api::dbname: nova_api
         nova::db::mysql_api::allowed_hosts:
           - '%'

puppet/services/sahara-base.yaml

@@ -53,7 +53,7 @@ outputs:
               - '/sahara'
         sahara::db::mysql::password: {get_param: SaharaPassword}
         sahara::db::mysql::user: sahara
-        sahara::db::mysql::host: {get_param: [EndpointMap, MysqlNoBracketsInternal, host]}
+        sahara::db::mysql::host: {get_param: [EndpointMap, MysqlInternal, host_nobrackets]}
         sahara::db::mysql::dbname: sahara
         sahara::db::mysql::allowed_hosts:
           - '%'