From 00c17b368f8ff42cb1e2e672ba1ddd702b7b6095 Mon Sep 17 00:00:00 2001 From: Alan Bishop Date: Sun, 8 Dec 2019 05:47:09 -0800 Subject: [PATCH] Add ability to deploy glance at DCN/Edge sites This patch adds two new tripleo services that together support deploying the glance-api service at edge sites. The service uses the same glance database in the control plane, but allows other edge services (e.g. cinder and nova) to access a glance endpoint that is local to the edge site. A new GlanceApiEdge service is a minor variant of the GlanceApi service. The most significant change is it doesn't use the control plane VIP, but instead configures cinder and nova services to access the glance-api endpoint running on that edge node (not the VIP). A companion HAproxyEdge service supports scaling out DCN sites with larger (>3) number of nodes. Instead of deploying GlanceApiEdge on every node, the HAproxyEdge service configures a local haproxy to forward glance-api requests to the edge nodes running GlanceApiEdge. The HAproxyEdge is extensible. While this patch is only concerned with proxying glance-api, it can be extended to support additional proxy requirements as needs arise. blueprint: split-controlplane-glance-cache Change-Id: Id6c416b8c7b3b6314d935e3eeb8a3f114492cecd Depends-On: Ic8d652a5209219c96f795a8c18ceb457c6d9382a (cherry picked from commit 30ca49bf611d0c3b443df6e7a636628dee281303) --- .../glance-api-edge-container-puppet.yaml | 82 ++++++++++++ .../haproxy-edge-container-puppet.yaml | 123 ++++++++++++++++++ environments/dcn-hci.yaml | 10 ++ environments/dcn.yaml | 10 ++ network/service_net_map.j2.yaml | 1 + overcloud-resource-registry-puppet.j2.yaml | 2 + ...-glance-at-dcn-sites-6163b8f5333e31a7.yaml | 8 ++ roles/DistributedCompute.yaml | 6 + roles/DistributedComputeHCI.yaml | 6 + sample-env-generator/dcn.yaml | 6 + 10 files changed, 254 insertions(+) create mode 100644 deployment/glance/glance-api-edge-container-puppet.yaml create mode 100644 deployment/haproxy/haproxy-edge-container-puppet.yaml create mode 100644 releasenotes/notes/support-glance-at-dcn-sites-6163b8f5333e31a7.yaml diff --git a/deployment/glance/glance-api-edge-container-puppet.yaml b/deployment/glance/glance-api-edge-container-puppet.yaml new file mode 100644 index 0000000000..357e1556a4 --- /dev/null +++ b/deployment/glance/glance-api-edge-container-puppet.yaml @@ -0,0 +1,82 @@ +heat_template_version: rocky + +description: > + OpenStack Glance service for DCN/Edge configured with Puppet + +parameters: + ServiceData: + default: {} + description: Dictionary packing service data + type: json + ServiceNetMap: + default: {} + description: Mapping of service_name -> network name. Typically set + via parameter_defaults in the resource registry. This + mapping overrides those in ServiceNetMapDefaults. + type: json + DefaultPasswords: + default: {} + type: json + RoleName: + default: '' + description: Role name on which the service is applied + type: string + RoleParameters: + default: {} + description: Parameters specific to the role + type: json + EndpointMap: + default: {} + description: Mapping of service endpoint -> protocol. Typically set + via parameter_defaults in the resource registry. + type: json + EnableInternalTLS: + type: boolean + default: false + +conditions: + + internal_tls_enabled: {equals: [{get_param: EnableInternalTLS}, true]} + +resources: + + GlanceApiBase: + type: ./glance-api-container-puppet.yaml + properties: + ServiceData: {get_param: ServiceData} + ServiceNetMap: {get_param: ServiceNetMap} + DefaultPasswords: {get_param: DefaultPasswords} + EndpointMap: {get_param: EndpointMap} + RoleName: {get_param: RoleName} + RoleParameters: {get_param: RoleParameters} + +outputs: + + glance_api_edge_uri: + description: URI of the glance-api service runing at the edge site. + value: &glance_api_edge_uri + str_replace: + template: + "PROTOCOL://%{hiera('NETWORK')}:9292" + params: + PROTOCOL: + if: + - internal_tls_enabled + - https + - http + NETWORK: {get_param: [ServiceNetMap, GlanceApiEdgeNetwork]} + + role_data: + description: Role data for the Glance API role for DCN/Edge. + value: + map_merge: + - get_attr: [GlanceApiBase, role_data] + - service_name: glance_api_edge + service_config_settings: + map_merge: + - get_attr: [GlanceApiBase, role_data, service_config_settings] + - cinder_volume: + cinder::glance::glance_api_servers: *glance_api_edge_uri + nova_compute: + nova::glance_api_servers: *glance_api_edge_uri + diff --git a/deployment/haproxy/haproxy-edge-container-puppet.yaml b/deployment/haproxy/haproxy-edge-container-puppet.yaml new file mode 100644 index 0000000000..ad9e404f98 --- /dev/null +++ b/deployment/haproxy/haproxy-edge-container-puppet.yaml @@ -0,0 +1,123 @@ +heat_template_version: rocky + +description: > + OpenStack containerized HAproxy service for DCN/Edge deployments + +parameters: + ServiceData: + default: {} + description: Dictionary packing service data + type: json + ServiceNetMap: + default: {} + description: Mapping of service_name -> network name. Typically set + via parameter_defaults in the resource registry. This + mapping overrides those in ServiceNetMapDefaults. + type: json + DefaultPasswords: + default: {} + type: json + RoleName: + default: '' + description: Role name on which the service is applied + type: string + RoleParameters: + default: {} + description: Parameters specific to the role + type: json + EndpointMap: + default: {} + description: Mapping of service endpoint -> protocol. Typically set + via parameter_defaults in the resource registry. + type: json + EnableGlanceApiProxy: + default: true + description: Configure haproxy to forward glance-api requests to glance-api + services running at the edge site. + type: boolean + EnableInternalTLS: + type: boolean + default: false + +conditions: + + glance_api_proxy_enabled: {equals: [{get_param: EnableGlanceApiProxy}, true]} + internal_tls_enabled: {equals: [{get_param: EnableInternalTLS}, true]} + +resources: + + HAProxyBase: + type: ./haproxy-container-puppet.yaml + properties: + ServiceData: {get_param: ServiceData} + ServiceNetMap: {get_param: ServiceNetMap} + DefaultPasswords: {get_param: DefaultPasswords} + EndpointMap: {get_param: EndpointMap} + RoleName: {get_param: RoleName} + RoleParameters: {get_param: RoleParameters} + +outputs: + + role_data: + description: Role data for the HAproxy role for DCN/Edge. + value: + map_merge: + - get_attr: [HAProxyBase, role_data] + - service_name: haproxy_edge + config_settings: + map_merge: + - get_attr: [HAProxyBase, role_data, config_settings] + - tripleo::haproxy::public_virtual_ip: null + tripleo::haproxy::aodh: false + tripleo::haproxy::barbican: false + tripleo::haproxy::ceph_grafana: false + tripleo::haproxy::ceph_rgw: false + tripleo::haproxy::cinder: false + tripleo::haproxy::designate: false + tripleo::haproxy::docker_registry: false + tripleo::haproxy::ec2_api: false + tripleo::haproxy::ec2_api_metadata: false + tripleo::haproxy::etcd: false + - if: + - glance_api_proxy_enabled + - tripleo::haproxy::glance_api: true + glance_api_vip: + str_replace: + template: + "%{hiera('NETWORK')}" + params: + NETWORK: {get_param: [ServiceNetMap, GlanceApiEdgeNetwork]} + glance_api_node_ips: "%{alias('glance_api_edge_node_ips')}" + glance_api_node_names: "%{alias('glance_api_edge_node_names')}" + - tripleo::haproxy::glance_api: false + - tripleo::haproxy::gnocchi: false + tripleo::haproxy::heat_api: false + tripleo::haproxy::heat_cfn: false + tripleo::haproxy::horizon: false + tripleo::haproxy::ironic: false + tripleo::haproxy::ironic_inspector: false + tripleo::haproxy::keystone_admin: false + tripleo::haproxy::keystone_public: false + tripleo::haproxy::kubernetes_master: false + tripleo::haproxy::manila: false + tripleo::haproxy::mistral: false + tripleo::haproxy::mysql: false + tripleo::haproxy::neutron: false + tripleo::haproxy::nova_osapi: false + tripleo::haproxy::nova_metadata: false + tripleo::haproxy::nova_novncproxy: false + tripleo::haproxy::octavia: false + tripleo::haproxy::opendaylight: false + tripleo::haproxy::openshift_master: false + tripleo::haproxy::ovn_dbs: false + tripleo::haproxy::panko: false + tripleo::haproxy::placement: false + tripleo::haproxy::rabbitmq: false + tripleo::haproxy::redis: false + tripleo::haproxy::sahara: false + tripleo::haproxy::swift_proxy_server: false + tripleo::haproxy::tacker: false + tripleo::haproxy::trove: false + tripleo::haproxy::ui: false + tripleo::haproxy::zaqar_api: false + tripleo::haproxy::zaqar_ws: false diff --git a/environments/dcn-hci.yaml b/environments/dcn-hci.yaml index 3395b314a7..8132ef9bbf 100644 --- a/environments/dcn-hci.yaml +++ b/environments/dcn-hci.yaml @@ -22,6 +22,14 @@ parameter_defaults: # Type: string CinderVolumeCluster: dcn + # Enable Glance Image Cache + # Type: boolean + GlanceCacheEnabled: False + + # The upper limit on cache size, in bytes, after which the cache-pruner cleans up the image cache. + # Type: number + GlanceImageCacheMaxSize: 10737418240 + # Manage the network and related resources (subnets and segments) with either create, update, or delete operations (depending on the stack operation). Does not apply to ports which will always be managed as needed. Defaults to true. For multi-stack use cases where the network related resources have already been managed by a separate stack, this parameter can be set to false. # Type: boolean ManageNetworks: False @@ -35,4 +43,6 @@ resource_registry: OS::TripleO::Network::Ports::RedisVipPort: ../network/ports/noop.yaml OS::TripleO::Services::CinderVolume: ../deployment/cinder/cinder-volume-container-puppet.yaml OS::TripleO::Services::Etcd: ../../deployment/etcd/etcd-container-puppet.yaml + OS::TripleO::Services::GlanceApiEdge: ../deployment/glance/glance-api-edge-container-puppet.yaml + OS::TripleO::Services::HAproxyEdge: ../deployment/haproxy/haproxy-edge-container-puppet.yaml OS::TripleO::Services::NovaAZConfig: ../deployment/nova/nova-az-config.yaml diff --git a/environments/dcn.yaml b/environments/dcn.yaml index 8ec2b02798..12e1ee1bc4 100644 --- a/environments/dcn.yaml +++ b/environments/dcn.yaml @@ -9,6 +9,14 @@ # Environment file for deploying a remote site of distributed compute nodes # (DCN) in a separate stack (multi-stack) deployment. parameter_defaults: + # Enable Glance Image Cache + # Type: boolean + GlanceCacheEnabled: False + + # The upper limit on cache size, in bytes, after which the cache-pruner cleans up the image cache. + # Type: number + GlanceImageCacheMaxSize: 10737418240 + # Manage the network and related resources (subnets and segments) with either create, update, or delete operations (depending on the stack operation). Does not apply to ports which will always be managed as needed. Defaults to true. For multi-stack use cases where the network related resources have already been managed by a separate stack, this parameter can be set to false. # Type: boolean ManageNetworks: False @@ -20,4 +28,6 @@ parameter_defaults: resource_registry: OS::TripleO::Network::Ports::OVNDBsVipPort: ../network/ports/noop.yaml OS::TripleO::Network::Ports::RedisVipPort: ../network/ports/noop.yaml + OS::TripleO::Services::GlanceApiEdge: ../deployment/glance/glance-api-edge-container-puppet.yaml + OS::TripleO::Services::HAproxyEdge: ../deployment/haproxy/haproxy-edge-container-puppet.yaml OS::TripleO::Services::NovaAZConfig: ../deployment/nova/nova-az-config.yaml diff --git a/network/service_net_map.j2.yaml b/network/service_net_map.j2.yaml index 7749cbb9a8..67d2e41830 100644 --- a/network/service_net_map.j2.yaml +++ b/network/service_net_map.j2.yaml @@ -41,6 +41,7 @@ parameters: CinderApiNetwork: {{ _service_nets.get('internal_api', 'ctlplane') }} CinderIscsiNetwork: {{ _service_nets.get('storage', 'ctlplane') }} GlanceApiNetwork: {{ _service_nets.get('internal_api', 'ctlplane') }} + GlanceApiEdgeNetwork: {{ _service_nets.get('internal_api', 'ctlplane') }} IronicApiNetwork: ctlplane IronicNetwork: ctlplane IronicInspectorNetwork: ctlplane diff --git a/overcloud-resource-registry-puppet.j2.yaml b/overcloud-resource-registry-puppet.j2.yaml index cab8405678..9c9f03ccf3 100644 --- a/overcloud-resource-registry-puppet.j2.yaml +++ b/overcloud-resource-registry-puppet.j2.yaml @@ -331,6 +331,8 @@ resource_registry: # see https://review.opendev.org/#/c/563443 OS::TripleO::Docker::NeutronMl2PluginBase: deployment/neutron/neutron-plugin-ml2-ovn.yaml OS::TripleO::Services::Multipathd: OS::Heat::None + OS::TripleO::Services::GlanceApiEdge: OS::Heat::None + OS::TripleO::Services::HAproxyEdge: OS::Heat::None # Logging OS::TripleO::Services::Tmpwatch: deployment/logrotate/tmpwatch-install.yaml diff --git a/releasenotes/notes/support-glance-at-dcn-sites-6163b8f5333e31a7.yaml b/releasenotes/notes/support-glance-at-dcn-sites-6163b8f5333e31a7.yaml new file mode 100644 index 0000000000..089bc53413 --- /dev/null +++ b/releasenotes/notes/support-glance-at-dcn-sites-6163b8f5333e31a7.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + Add the ability to deploy the glance-api service at DCN/Edge sites. Glance + service at the Edge shares the same database as the Glance service in the + central control plane, but allows other services such as Cinder and Nova to + access a Glance endpoint that is local to the DCN/Edge site. + diff --git a/roles/DistributedCompute.yaml b/roles/DistributedCompute.yaml index 282e33920a..52d1dddf07 100644 --- a/roles/DistributedCompute.yaml +++ b/roles/DistributedCompute.yaml @@ -32,6 +32,12 @@ - OS::TripleO::Services::ComputeNeutronOvsAgent - OS::TripleO::Services::Docker - OS::TripleO::Services::Etcd + # Include the GlanceApiEdge -OR- the HAproxyEdge service, but not both. + # Deploy GlanceApiEdge on 2-3 nodes for HA, and HAproxyEdge for each + # additional node. The HAproxyEdge service forwards glance-api requests + # to the GlanceApiEdge nodes. + - OS::TripleO::Services::GlanceApiEdge + # OS::TripleO::Services::HAproxyEdge - OS::TripleO::Services::IpaClient - OS::TripleO::Services::Ipsec - OS::TripleO::Services::Iscsid diff --git a/roles/DistributedComputeHCI.yaml b/roles/DistributedComputeHCI.yaml index 06dbf0f0c9..2443bd7e2c 100644 --- a/roles/DistributedComputeHCI.yaml +++ b/roles/DistributedComputeHCI.yaml @@ -40,6 +40,12 @@ - OS::TripleO::Services::ComputeNeutronOvsAgent - OS::TripleO::Services::Docker - OS::TripleO::Services::Etcd + # Include the GlanceApiEdge -OR- the HAproxyEdge service, but not both. + # Deploy GlanceApiEdge on 2-3 nodes for HA, and HAproxyEdge for each + # additional node. The HAproxyEdge service forwards glance-api requests + # to the GlanceApiEdge nodes. + - OS::TripleO::Services::GlanceApiEdge + # OS::TripleO::Services::HAproxyEdge - OS::TripleO::Services::IpaClient - OS::TripleO::Services::Ipsec - OS::TripleO::Services::Iscsid diff --git a/sample-env-generator/dcn.yaml b/sample-env-generator/dcn.yaml index d1d0ee835f..4413911561 100644 --- a/sample-env-generator/dcn.yaml +++ b/sample-env-generator/dcn.yaml @@ -5,6 +5,10 @@ environments: Environment file for deploying a remote site of distributed compute nodes (DCN) in a separate stack (multi-stack) deployment. files: &dcn_files + deployment/glance/glance-api-container-puppet.yaml: + parameters: + - GlanceCacheEnabled + - GlanceImageCacheMaxSize deployment/nova/nova-az-config.yaml: parameters: - NovaComputeAvailabilityZone @@ -16,6 +20,8 @@ environments: resource_registry: &dcn_resource_registry OS::TripleO::Network::Ports::RedisVipPort: ../network/ports/noop.yaml OS::TripleO::Network::Ports::OVNDBsVipPort: ../network/ports/noop.yaml + OS::TripleO::Services::GlanceApiEdge: ../deployment/glance/glance-api-edge-container-puppet.yaml + OS::TripleO::Services::HAproxyEdge: ../deployment/haproxy/haproxy-edge-container-puppet.yaml OS::TripleO::Services::NovaAZConfig: ../deployment/nova/nova-az-config.yaml children: - name: dcn-hci