From 32cce5f150d0c618584cf65550343f435ea6afb1 Mon Sep 17 00:00:00 2001 From: Damien Ciabrini Date: Tue, 12 Dec 2017 11:07:34 +0000 Subject: [PATCH] Fix Redis TLS setup, including replication traffic This patch reverts the revert of Redis TLS [1], and fixes the encryption of Redis replication traffic for HA deployments. In order to encrypt replication traffic, Redis is configured to drive outgoing replication traffic to a stunnel endpoint on . Stunnel then manages the encryption up to the peer Redis master. Likewise, slave Redis nodes advertise themselves as coming from in order to let the Master initiate connection the Slave over its own stunnel endpoint, should it needs to. Each redis node is assigned a unique replication port, and has dedicated stunnels to each one of its peer. This port mapping info is used by the redis resource agent to manage A/P failover. The regular Redis port is unchanged, so Redis clients (OpenStack services, HAproxy, CLI, firewall) are not impacted by this change. Only SELinux needs to be adapted. [1] I37501c4c983c87e3a38841272eb176ebbe626a65 Change-Id: I6cc818973fab25b4cd6f7a0d040aaa05a35c5bb1 Related-bug: #1737707 --- manifests/certmonger/redis.pp | 72 ++++ manifests/haproxy.pp | 17 +- manifests/profile/base/aodh/evaluator.pp | 14 +- .../profile/base/ceilometer/agent/polling.pp | 13 +- manifests/profile/base/certmonger_user.pp | 9 + manifests/profile/base/database/redis.pp | 71 +++- manifests/profile/base/gnocchi/api.pp | 6 +- manifests/profile/pacemaker/database/redis.pp | 156 ++++++++- .../pacemaker/database/redis_bundle.pp | 311 ++++++++++++++---- 9 files changed, 585 insertions(+), 84 deletions(-) create mode 100644 manifests/certmonger/redis.pp diff --git a/manifests/certmonger/redis.pp b/manifests/certmonger/redis.pp new file mode 100644 index 000000000..1b3b119ea --- /dev/null +++ b/manifests/certmonger/redis.pp @@ -0,0 +1,72 @@ +# Copyright 2017 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# == Class: tripleo::certmonger::redis +# +# Request a certificate for RabbitMQ and do the necessary setup. +# +# === Parameters +# +# [*hostname*] +# The hostname of the node. this will be set in the CN of the certificate. +# +# [*service_certificate*] +# The path to the certificate that will be used for TLS in this service. +# +# [*service_key*] +# The path to the key that will be used for TLS in this service. +# +# [*certmonger_ca*] +# (Optional) The CA that certmonger will use to generate the certificates. +# Defaults to hiera('certmonger_ca', 'local'). +# +# [*postsave_cmd*] +# (Optional) Specifies the command to execute after requesting a certificate. +# If nothing is given, it will default to: "systemctl restart ${service name}" +# Defaults to undef. +# +# [*principal*] +# (Optional) The service principal that is set for the service in kerberos. +# Defaults to undef +# +class tripleo::certmonger::redis ( + $hostname, + $service_certificate, + $service_key, + $certmonger_ca = hiera('certmonger_ca', 'local'), + $postsave_cmd = undef, + $principal = undef, +) { + include ::certmonger + + certmonger_certificate { 'redis' : + ensure => 'present', + certfile => $service_certificate, + keyfile => $service_key, + hostname => $hostname, + dnsname => $hostname, + principal => $principal, + postsave_cmd => $postsave_cmd, + ca => $certmonger_ca, + wait => true, + require => Class['::certmonger'], + } + + file { $service_certificate : + require => Certmonger_certificate['redis'], + } + file { $service_key : + require => Certmonger_certificate['redis'], + } +} diff --git a/manifests/haproxy.pp b/manifests/haproxy.pp index 6328fb1f6..21f3a35ba 100644 --- a/manifests/haproxy.pp +++ b/manifests/haproxy.pp @@ -1381,11 +1381,19 @@ class tripleo::haproxy ( } if $redis { - if $redis_password { - $redis_tcp_check_options = ["send AUTH\\ ${redis_password}\\r\\n"] + if $enable_internal_tls { + $redis_tcp_check_ssl_options = ['connect ssl'] + $redis_ssl_member_options = ['check-ssl', "ca-file ${ca_bundle}"] } else { - $redis_tcp_check_options = [] + $redis_tcp_check_ssl_options = [] + $redis_ssl_member_options = [] } + if $redis_password { + $redis_tcp_check_password_options = ["send AUTH\\ ${redis_password}\\r\\n"] + } else { + $redis_tcp_check_password_options = [] + } + $redis_tcp_check_options = union($redis_tcp_check_ssl_options, $redis_tcp_check_password_options) haproxy::listen { 'redis': bind => $redis_bind_opts, options => { @@ -1405,7 +1413,8 @@ class tripleo::haproxy ( ports => '6379', ipaddresses => hiera('redis_node_ips', $controller_hosts_real), server_names => hiera('redis_node_names', $controller_hosts_names_real), - options => union($haproxy_member_options, ['on-marked-down shutdown-sessions']), + options => union($haproxy_member_options, ['on-marked-down shutdown-sessions'], $redis_ssl_member_options), + verifyhost => false, } if $manage_firewall { include ::tripleo::firewall diff --git a/manifests/profile/base/aodh/evaluator.pp b/manifests/profile/base/aodh/evaluator.pp index 1b25b3760..9b3462f94 100644 --- a/manifests/profile/base/aodh/evaluator.pp +++ b/manifests/profile/base/aodh/evaluator.pp @@ -18,20 +18,30 @@ # # === Parameters # +# [*enable_internal_tls*] +# (Optional) Whether TLS in the internal network is enabled or not. +# Defaults to hiera('enable_internal_tls', false) +# # [*step*] # (Optional) The current step in deployment. See tripleo-heat-templates # for more details. # Defaults to hiera('step') # class tripleo::profile::base::aodh::evaluator ( - $step = Integer(hiera('step')), + $enable_internal_tls = hiera('enable_internal_tls', false), + $step = Integer(hiera('step')), ) { include ::tripleo::profile::base::aodh + if $enable_internal_tls { + $tls_query_param = '?ssl=true' + } else { + $tls_query_param = '' + } if $step >= 4 { class { '::aodh::evaluator': - coordination_url => join(['redis://:', hiera('aodh_redis_password'), '@', normalize_ip_for_uri(hiera('redis_vip')), ':6379/']), + coordination_url => join(['redis://:', hiera('aodh_redis_password'), '@', normalize_ip_for_uri(hiera('redis_vip')), ':6379/', $tls_query_param]), } } diff --git a/manifests/profile/base/ceilometer/agent/polling.pp b/manifests/profile/base/ceilometer/agent/polling.pp index 84f5e46c4..043b5cd9a 100644 --- a/manifests/profile/base/ceilometer/agent/polling.pp +++ b/manifests/profile/base/ceilometer/agent/polling.pp @@ -26,6 +26,10 @@ # (Optional) Use compute namespace for polling agent. # Defaults to false. # +# [*enable_internal_tls*] +# (Optional) Whether TLS in the internal network is enabled or not. +# Defaults to hiera('enable_internal_tls', false) +# # [*ipmi_namespace*] # (Optional) Use ipmi namespace for polling agent. # Defaults to false. @@ -44,6 +48,7 @@ class tripleo::profile::base::ceilometer::agent::polling ( $central_namespace = hiera('central_namespace', false), $compute_namespace = hiera('compute_namespace', false), + $enable_internal_tls = hiera('enable_internal_tls', false), $ipmi_namespace = hiera('ipmi_namespace', false), $ceilometer_redis_password = hiera('ceilometer_redis_password', undef), $redis_vip = hiera('redis_vip', undef), @@ -55,13 +60,19 @@ class tripleo::profile::base::ceilometer::agent::polling ( include ::tripleo::profile::base::ceilometer::upgrade } + if $enable_internal_tls { + $tls_query_param = '?ssl=true' + } else { + $tls_query_param = '' + } + if $step >= 4 { include ::ceilometer::agent::auth class { '::ceilometer::agent::polling': central_namespace => $central_namespace, compute_namespace => $compute_namespace, ipmi_namespace => $ipmi_namespace, - coordination_url => join(['redis://:', $ceilometer_redis_password, '@', normalize_ip_for_uri($redis_vip), ':6379/']), + coordination_url => join(['redis://:', $ceilometer_redis_password, '@', normalize_ip_for_uri($redis_vip), ':6379/', $tls_query_param]), } } } diff --git a/manifests/profile/base/certmonger_user.pp b/manifests/profile/base/certmonger_user.pp index 8f6d26540..8b3ce6594 100644 --- a/manifests/profile/base/certmonger_user.pp +++ b/manifests/profile/base/certmonger_user.pp @@ -78,6 +78,11 @@ # it will create. # Defaults to hiera('tripleo::profile::base::rabbitmq::certificate_specs', {}). # +# [*redis_certificate_specs*] +# (Optional) The specifications to give to certmonger for the certificate(s) +# it will create. +# Defaults to hiera('redis_certificate_specs', {}). +# # [*etcd_certificate_specs*] # (Optional) The specifications to give to certmonger for the certificate(s) # it will create. @@ -108,6 +113,7 @@ class tripleo::profile::base::certmonger_user ( $mongodb_certificate_specs = hiera('mongodb_certificate_specs',{}), $mysql_certificate_specs = hiera('tripleo::profile::base::database::mysql::certificate_specs', {}), $rabbitmq_certificate_specs = hiera('tripleo::profile::base::rabbitmq::certificate_specs', {}), + $redis_certificate_specs = hiera('redis_certificate_specs', {}), $etcd_certificate_specs = hiera('tripleo::profile::base::etcd::certificate_specs', {}), $odl_certificate_specs = hiera('tripleo::profile::base::neutron::opendaylight::certificate_specs', {}), $ovs_certificate_specs = hiera('tripleo::profile::base::neutron::plugins::ovs::opendaylight::certificate_specs', {}), @@ -155,6 +161,9 @@ class tripleo::profile::base::certmonger_user ( unless empty($rabbitmq_certificate_specs) { ensure_resource('class', 'tripleo::certmonger::rabbitmq', $rabbitmq_certificate_specs) } + unless empty($redis_certificate_specs) { + ensure_resource('class', 'tripleo::certmonger::redis', $redis_certificate_specs) + } unless empty($etcd_certificate_specs) { ensure_resource('class', 'tripleo::certmonger::etcd', $etcd_certificate_specs) } diff --git a/manifests/profile/base/database/redis.pp b/manifests/profile/base/database/redis.pp index e35735935..8d4ed94ca 100644 --- a/manifests/profile/base/database/redis.pp +++ b/manifests/profile/base/database/redis.pp @@ -22,6 +22,26 @@ # (Optional) Hostname of Redis master # Defaults to hiera('bootstrap_nodeid') # +# [*certificate_specs*] +# (Optional) The specifications to give to certmonger for the certificate(s) +# it will create. +# Example with hiera: +# redis_certificate_specs: +# hostname: +# service_certificate: +# service_key: +# principal: "haproxy/" +# Defaults to hiera('redis_certificate_specs', {}). +# +# [*enable_internal_tls*] +# (Optional) Whether TLS in the internal network is enabled or not. +# Defaults to hiera('enable_internal_tls', false) +# +# [*redis_network*] +# (Optional) The network name where the redis endpoint is listening on. +# This is set by t-h-t. +# Defaults to hiera('redis_network', undef) +# # [*redis_node_ips*] # (Optional) List of Redis node ips # Defaults to hiera('redis_node_ips') @@ -31,12 +51,57 @@ # for more details. # Defaults to hiera('step') # +# [*tls_proxy_bind_ip*] +# IP on which the TLS proxy will listen on. Required only if +# enable_internal_tls is set. +# Defaults to undef +# +# [*tls_proxy_fqdn*] +# fqdn on which the tls proxy will listen on. required only used if +# enable_internal_tls is set. +# defaults to undef +# +# [*tls_proxy_port*] +# port on which the tls proxy will listen on. Only used if +# enable_internal_tls is set. +# defaults to 6379 +# class tripleo::profile::base::database::redis ( - $bootstrap_nodeid = hiera('bootstrap_nodeid'), - $redis_node_ips = hiera('redis_node_ips'), - $step = Integer(hiera('step')), + $bootstrap_nodeid = hiera('bootstrap_nodeid'), + $certificate_specs = hiera('redis_certificate_specs', {}), + $enable_internal_tls = hiera('enable_internal_tls', false), + $redis_network = hiera('redis_network', undef), + $redis_node_ips = hiera('redis_node_ips'), + $step = Integer(hiera('step')), + $tls_proxy_bind_ip = undef, + $tls_proxy_fqdn = undef, + $tls_proxy_port = 6379, ) { if $step >= 2 { + if $enable_internal_tls { + if !$redis_network { + fail('redis_network is not set in the hieradata.') + } + if !$tls_proxy_bind_ip { + fail('tls_proxy_bind_ip is not set in the hieradata.') + } + if !$tls_proxy_fqdn { + fail('tls_proxy_fqdn is required if internal TLS is enabled.') + } + $tls_certfile = $certificate_specs['service_certificate'] + $tls_keyfile = $certificate_specs['service_key'] + + include ::tripleo::stunnel + + ::tripleo::stunnel::service_proxy { 'redis': + accept_host => $tls_proxy_bind_ip, + accept_port => $tls_proxy_port, + connect_port => $tls_proxy_port, + certificate => $tls_certfile, + key => $tls_keyfile, + notify => Class['::redis'], + } + } if downcase($bootstrap_nodeid) == $::hostname { $slaveof = undef } else { diff --git a/manifests/profile/base/gnocchi/api.pp b/manifests/profile/base/gnocchi/api.pp index c031612dd..8a7d780cc 100644 --- a/manifests/profile/base/gnocchi/api.pp +++ b/manifests/profile/base/gnocchi/api.pp @@ -94,9 +94,11 @@ class tripleo::profile::base::gnocchi::api ( } $tls_certfile = $certificates_specs["httpd-${gnocchi_network}"]['service_certificate'] $tls_keyfile = $certificates_specs["httpd-${gnocchi_network}"]['service_key'] + $tls_query_param = '?ssl=true' } else { $tls_certfile = undef $tls_keyfile = undef + $tls_query_param = '' } if $step >= 4 or ($step >= 3 and $sync_db) { @@ -122,12 +124,12 @@ class tripleo::profile::base::gnocchi::api ( } class { '::gnocchi::storage': - coordination_url => join(['redis://:', $gnocchi_redis_password, '@', normalize_ip_for_uri($redis_vip), ':6379/']), + coordination_url => join(['redis://:', $gnocchi_redis_password, '@', normalize_ip_for_uri($redis_vip), ':6379/', $tls_query_param]), } if $incoming_storage_driver == 'redis' { class { '::gnocchi::storage::incoming::redis': - redis_url => join(['redis://:', $gnocchi_redis_password, '@', normalize_ip_for_uri($redis_vip), ':6379/']), + redis_url => join(['redis://:', $gnocchi_redis_password, '@', normalize_ip_for_uri($redis_vip), ':6379/', $tls_query_param]), } } diff --git a/manifests/profile/pacemaker/database/redis.pp b/manifests/profile/pacemaker/database/redis.pp index bc91be77e..fbd4ea657 100644 --- a/manifests/profile/pacemaker/database/redis.pp +++ b/manifests/profile/pacemaker/database/redis.pp @@ -22,6 +22,21 @@ # (Optional) The hostname of the node responsible for bootstrapping tasks # Defaults to hiera('redis_short_bootstrap_node_name') # +# [*certificate_specs*] +# (Optional) The specifications to give to certmonger for the certificate(s) +# it will create. +# Example with hiera: +# redis_certificate_specs: +# hostname: +# service_certificate: +# service_key: +# principal: "haproxy/" +# Defaults to hiera('redis_certificate_specs', {}). +# +# [*enable_internal_tls*] +# (Optional) Whether TLS in the internal network is enabled or not. +# Defaults to hiera('enable_internal_tls', false) +# # [*enable_load_balancer*] # (Optional) Whether load balancing is enabled for this cluster # Defaults to hiera('enable_load_balancer', true) @@ -39,16 +54,62 @@ # https://github.com/arioch/puppet-redis/pull/192. Set redis::ulimit via hiera # to control this limit. # +# [*redis_network*] +# (Optional) The network name where the redis endpoint is listening on. +# This is set by t-h-t. +# Defaults to hiera('redis_network', undef) +# # [*pcs_tries*] # (Optional) The number of times pcs commands should be retried. # Defaults to hiera('pcs_tries', 20) # +# [*extra_config_file*] +# (Optional) When TLS proxy is in use, name of a host-specific Redis +# config file that configures tunnel connection. +# This is set by t-h-t. +# Defaults to '/etc/redis-tls.conf' +# +# [*tls_tunnel_local_name*] +# (Optional) When TLS proxy is in use, name of the localhost to forward +# unencryption Redis traffic to. +# This is set by t-h-t. +# Defaults to 'localhost' +# +# [*tls_tunnel_base_port*] +# (Optional) When TLS proxy is in use, a base integer value that is used +# to generate a unique port number for each peer in the Redis cluster. +# Defaults to '6660' +# +# [*tls_proxy_bind_ip*] +# IP on which the TLS proxy will listen on. Required only if +# enable_internal_tls is set. +# Defaults to undef +# +# [*tls_proxy_fqdn*] +# fqdn on which the tls proxy will listen on. required only used if +# enable_internal_tls is set. +# defaults to undef +# +# [*tls_proxy_port*] +# port on which the tls proxy will listen on. Only used if +# enable_internal_tls is set. +# defaults to 6379 +# class tripleo::profile::pacemaker::database::redis ( - $bootstrap_node = hiera('redis_short_bootstrap_node_name'), - $enable_load_balancer = hiera('enable_load_balancer', true), - $step = Integer(hiera('step')), - $redis_file_limit = undef, - $pcs_tries = hiera('pcs_tries', 20), + $certificate_specs = hiera('redis_certificate_specs', {}), + $enable_internal_tls = hiera('enable_internal_tls', false), + $bootstrap_node = hiera('redis_short_bootstrap_node_name'), + $enable_load_balancer = hiera('enable_load_balancer', true), + $step = Integer(hiera('step')), + $redis_file_limit = undef, + $redis_network = hiera('redis_network', undef), + $pcs_tries = hiera('pcs_tries', 20), + $extra_config_file = '/etc/redis-tls.conf', + $tls_tunnel_local_name = 'localhost', + $tls_tunnel_base_port = 6660, + $tls_proxy_bind_ip = undef, + $tls_proxy_fqdn = undef, + $tls_proxy_port = 6379, ) { if $::hostname == downcase($bootstrap_node) { $pacemaker_master = true @@ -56,7 +117,84 @@ class tripleo::profile::pacemaker::database::redis ( $pacemaker_master = false } + if $enable_internal_tls { + if !$redis_network { + fail('redis_network is not set in the hieradata.') + } + if !$tls_proxy_bind_ip { + fail('tls_proxy_bind_ip is not set in the hieradata.') + } + if !$tls_proxy_fqdn { + fail('tls_proxy_fqdn is required if internal TLS is enabled.') + } + + $redis_node_names = hiera('redis_short_node_names', [$::hostname]) + $redis_node_ips = hiera('redis_node_ips', [$tls_proxy_bind_ip]) + + # keep a mapping of [node name, node ip, replication port] + $replication_tuples = zip($redis_node_names, $redis_node_ips).map |$index, $pair| { + $pair.concat($tls_tunnel_base_port+$index) + } + } else { + $replication_tuples = [] + } + if $step >= 1 { + if $enable_internal_tls { + $tls_certfile = $certificate_specs['service_certificate'] + $tls_keyfile = $certificate_specs['service_key'] + + include ::tripleo::stunnel + + # encrypted endpoint for incoming redis service + ::tripleo::stunnel::service_proxy { 'redis': + accept_host => $tls_proxy_bind_ip, + accept_port => $tls_proxy_port, + connect_host => $tls_tunnel_local_name, + connect_port => $tls_proxy_port, + certificate => $tls_certfile, + key => $tls_keyfile, + notify => Class['::redis'], + } + + # encrypted endpoints for outgoing redis replication traffic + $redis_peers = $replication_tuples.filter |$tuple| {$tuple[1] != $tls_proxy_bind_ip} + $redis_peers.each |$tuple| { + ::tripleo::stunnel::service_proxy { "redis_peer_${tuple[2]}": + client => 'yes', + accept_host => $tls_tunnel_local_name, + accept_port => $tuple[2], + connect_host => $tuple[1], + connect_port => $tls_proxy_port, + certificate => $tls_certfile, + key => $tls_keyfile, + notify => Class['::redis'], + } + } + + # redis slave advertise itself as running on a specific + # that uniquely identifies it. This value is + # used by the master as is, and points the the outgoing stunnel + # endpoint to target this slave. + + $local_tuple = $replication_tuples.filter |$tuple| { + $tuple[1] == $tls_proxy_bind_ip + } + if length($local_tuple)!=1 { + fail("could not determine local TLS replication port (local ip: '${tls_proxy_bind_ip}', assigned ports: '${replication_tuples}')") + } + + # NOTE: config parameters slave-announce-* are not exposed by + # puppet-redis, so for now we configure them via an additional + # host-specific config file + File {"${extra_config_file}": + ensure => present, + content => "# Host-specific configuration for TLS +slave-announce-ip ${tls_tunnel_local_name} +slave-announce-port ${local_tuple[0][2]} +", + } + } # If the old hiera key exists we use that to set the ulimit in order not to break # operators which set it. We might remove this in a later release (post pike anyway) $old_redis_file_limit = hiera('redis_file_limit', undef) @@ -85,11 +223,17 @@ class tripleo::profile::pacemaker::database::redis ( node => $::hostname, } if $pacemaker_master { + if length($replication_tuples)>1 { + $tunnel_map = $replication_tuples.map |$tuple| {"${tuple[0]}:${tuple[2]}"} + $tunnel_opt = " tunnel_port_map='${tunnel_map.join(';')}' tunnel_host='${tls_tunnel_local_name}'" + } else { + $tunnel_opt='' + } pacemaker::resource::ocf { 'redis': ocf_agent_name => 'heartbeat:redis', master_params => '', meta_params => 'notify=true ordered=true interleave=true', - resource_params => 'wait_last_known_master=true', + resource_params => "wait_last_known_master=true${tunnel_opt}", op_params => 'start timeout=200s stop timeout=200s', tries => $pcs_tries, location_rule => { diff --git a/manifests/profile/pacemaker/database/redis_bundle.pp b/manifests/profile/pacemaker/database/redis_bundle.pp index 1e144f973..2b72ddc69 100644 --- a/manifests/profile/pacemaker/database/redis_bundle.pp +++ b/manifests/profile/pacemaker/database/redis_bundle.pp @@ -39,13 +39,74 @@ # for more details. # Defaults to hiera('step') # +# [*certificate_specs*] +# (Optional) The specifications to give to certmonger for the certificate(s) +# it will create. +# Example with hiera: +# redis_certificate_specs: +# hostname: +# service_certificate: +# service_key: +# principal: "haproxy/" +# Defaults to hiera('redis_certificate_specs', {}). +# +# [*enable_internal_tls*] +# (Optional) Whether TLS in the internal network is enabled or not. +# Defaults to hiera('enable_internal_tls', false) +# +# [*redis_network*] +# (Optional) The network name where the redis endpoint is listening on. +# This is set by t-h-t. +# Defaults to hiera('redis_network', undef) +# +# [*extra_config_file*] +# (Optional) When TLS proxy is in use, name of a host-specific Redis +# config file that configures tunnel connection. +# This is set by t-h-t. +# Defaults to '/etc/redis-tls.conf' +# +# [*tls_tunnel_local_name*] +# (Optional) When TLS proxy is in use, name of the localhost to forward +# unencryption Redis traffic to. +# This is set by t-h-t. +# Defaults to 'localhost' +# +# [*tls_tunnel_base_port*] +# (Optional) When TLS proxy is in use, a base integer value that is used +# to generate a unique port number for each peer in the Redis cluster. +# Defaults to '6660' +# +# [*tls_proxy_bind_ip*] +# IP on which the TLS proxy will listen on. Required only if +# enable_internal_tls is set. +# Defaults to undef +# +# [*tls_proxy_fqdn*] +# fqdn on which the tls proxy will listen on. required only used if +# enable_internal_tls is set. +# defaults to undef +# +# [*tls_proxy_port*] +# port on which the tls proxy will listen on. Only used if +# enable_internal_tls is set. +# defaults to 6379 +# # class tripleo::profile::pacemaker::database::redis_bundle ( + $certificate_specs = hiera('redis_certificate_specs', {}), + $enable_internal_tls = hiera('enable_internal_tls', false), $bootstrap_node = hiera('redis_short_bootstrap_node_name'), $redis_docker_image = hiera('tripleo::profile::pacemaker::database::redis_bundle::redis_docker_image', undef), $redis_docker_control_port = hiera('tripleo::profile::pacemaker::database::redis_bundle::control_port', '3124'), $pcs_tries = hiera('pcs_tries', 20), $step = Integer(hiera('step')), + $redis_network = hiera('redis_network', undef), + $extra_config_file = '/etc/redis-tls.conf', + $tls_tunnel_local_name = 'localhost', + $tls_tunnel_base_port = 6660, + $tls_proxy_bind_ip = undef, + $tls_proxy_fqdn = undef, + $tls_proxy_port = 6379, ) { if $::hostname == downcase($bootstrap_node) { $pacemaker_master = true @@ -53,7 +114,99 @@ class tripleo::profile::pacemaker::database::redis_bundle ( $pacemaker_master = false } - include ::tripleo::profile::base::database::redis + if $enable_internal_tls { + if !$redis_network { + fail('redis_network is not set in the hieradata.') + } + if !$tls_proxy_bind_ip { + fail('tls_proxy_bind_ip is not set in the hieradata.') + } + if !$tls_proxy_fqdn { + fail('tls_proxy_fqdn is required if internal TLS is enabled.') + } + + $redis_node_names = hiera('redis_short_node_names', [$::hostname]) + $redis_node_ips = hiera('redis_node_ips', [$tls_proxy_bind_ip]) + + # keep a mapping of [node name, node ip, replication port] + $replication_tuples = zip($redis_node_names, $redis_node_ips).map |$index, $pair| { + $pair.concat($tls_tunnel_base_port+$index) + } + } else { + $replication_tuples = [] + } + + if $step >= 1 { + if $enable_internal_tls { + $tls_certfile = $certificate_specs['service_certificate'] + $tls_keyfile = $certificate_specs['service_key'] + + include ::tripleo::stunnel + + # encrypted endpoint for incoming redis service + ::tripleo::stunnel::service_proxy { 'redis': + accept_host => $tls_proxy_bind_ip, + accept_port => $tls_proxy_port, + connect_host => $tls_tunnel_local_name, + connect_port => $tls_proxy_port, + certificate => $tls_certfile, + key => $tls_keyfile, + notify => Class['::redis'], + } + + # encrypted endpoints for outgoing redis replication traffic + $redis_peers = $replication_tuples.filter |$tuple| {$tuple[1] != $tls_proxy_bind_ip} + $redis_peers.each |$tuple| { + ::tripleo::stunnel::service_proxy { "redis_peer_${tuple[2]}": + client => 'yes', + accept_host => $tls_tunnel_local_name, + accept_port => $tuple[2], + connect_host => $tuple[1], + connect_port => $tls_proxy_port, + certificate => $tls_certfile, + key => $tls_keyfile, + notify => Class['::redis'], + } + } + + # redis slave advertise itself as running on a specific + # that uniquely identifies it. This value is + # used by the master as is, and points the the outgoing stunnel + # endpoint to target this slave. + + $local_tuple = $replication_tuples.filter |$tuple| { + $tuple[1] == $tls_proxy_bind_ip + } + if length($local_tuple)!=1 { + fail("could not determine local TLS replication port (local ip: '${tls_proxy_bind_ip}', assigned ports: '${replication_tuples}')") + } + + # NOTE: config parameters slave-announce-* are not exposed by + # puppet-redis, so for now we configure them via an additional + # host-specific config file + File {"${extra_config_file}": + ensure => present, + # owner => $::redis::config_owner, + # group => $::redis::config_group, + # mode => $::redis::config_file_mode, + content => "# Host-specific configuration for TLS +slave-announce-ip ${tls_tunnel_local_name} +slave-announce-port ${local_tuple[0][2]} +", + } + } + # If the old hiera key exists we use that to set the ulimit in order not to break + # operators which set it. We might remove this in a later release (post pike anyway) + $old_redis_file_limit = hiera('redis_file_limit', undef) + if $old_redis_file_limit != undef { + warning('redis_file_limit parameter is deprecated, use redis::ulimit in hiera.') + class { '::redis': + ulimit => $old_redis_file_limit, + } + } else { + include ::redis + } + } if $step >= 2 { if $pacemaker_master { @@ -69,6 +222,88 @@ class tripleo::profile::pacemaker::database::redis_bundle ( } } + $storage_maps = { + 'redis-cfg-files' => { + 'source-dir' => '/var/lib/kolla/config_files/redis.json', + 'target-dir' => '/var/lib/kolla/config_files/config.json', + 'options' => 'ro', + }, + 'redis-cfg-data-redis' => { + 'source-dir' => '/var/lib/config-data/puppet-generated/redis/', + 'target-dir' => '/var/lib/kolla/config_files/src', + 'options' => 'ro', + }, + 'redis-hosts' => { + 'source-dir' => '/etc/hosts', + 'target-dir' => '/etc/hosts', + 'options' => 'ro', + }, + 'redis-localtime' => { + 'source-dir' => '/etc/localtime', + 'target-dir' => '/etc/localtime', + 'options' => 'ro', + }, + 'redis-lib' => { + 'source-dir' => '/var/lib/redis', + 'target-dir' => '/var/lib/redis', + 'options' => 'rw', + }, + 'redis-log' => { + 'source-dir' => '/var/log/redis', + 'target-dir' => '/var/log/redis', + 'options' => 'rw', + }, + 'redis-run' => { + 'source-dir' => '/var/run/redis', + 'target-dir' => '/var/run/redis', + 'options' => 'rw', + }, + # TODO check whether those tls mappings are necessary + 'redis-pki-extracted' => { + 'source-dir' => '/etc/pki/ca-trust/extracted', + 'target-dir' => '/etc/pki/ca-trust/extracted', + 'options' => 'ro', + }, + 'redis-pki-ca-bundle-crt' => { + 'source-dir' => '/etc/pki/tls/certs/ca-bundle.crt', + 'target-dir' => '/etc/pki/tls/certs/ca-bundle.crt', + 'options' => 'ro', + }, + 'redis-pki-ca-bundle-trust-crt' => { + 'source-dir' => '/etc/pki/tls/certs/ca-bundle.trust.crt', + 'target-dir' => '/etc/pki/tls/certs/ca-bundle.trust.crt', + 'options' => 'ro', + }, + 'redis-pki-cert' => { + 'source-dir' => '/etc/pki/tls/cert.pem', + 'target-dir' => '/etc/pki/tls/cert.pem', + 'options' => 'ro', + }, + 'redis-dev-log' => { + 'source-dir' => '/dev/log', + 'target-dir' => '/dev/log', + 'options' => 'rw', + }, + } + + if $enable_internal_tls { + $redis_storage_maps_tls = { + 'redis-pki-gcomm-key' => { + 'source-dir' => '/etc/pki/tls/private/redis.key', + 'target-dir' => '/var/lib/kolla/config_files/src-tls/etc/pki/tls/private/redis.key', + 'options' => 'ro', + }, + 'redis-pki-gcomm-cert' => { + 'source-dir' => '/etc/pki/tls/certs/redis.crt', + 'target-dir' => '/var/lib/kolla/config_files/src-tls/etc/pki/tls/certs/redis.crt', + 'options' => 'ro', + }, + } + $storage_maps_tls = $redis_storage_maps_tls + } else { + $storage_maps_tls = {} + } + pacemaker::resource::bundle { 'redis-bundle': image => $redis_docker_image, replicas => $redis_nodes_count, @@ -82,73 +317,18 @@ class tripleo::profile::pacemaker::database::redis_bundle ( options => '--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS', run_command => '/bin/bash /usr/local/bin/kolla_start', network => "control-port=${redis_docker_control_port}", - storage_maps => { - 'redis-cfg-files' => { - 'source-dir' => '/var/lib/kolla/config_files/redis.json', - 'target-dir' => '/var/lib/kolla/config_files/config.json', - 'options' => 'ro', - }, - 'redis-cfg-data-redis' => { - 'source-dir' => '/var/lib/config-data/puppet-generated/redis/', - 'target-dir' => '/var/lib/kolla/config_files/src', - 'options' => 'ro', - }, - 'redis-hosts' => { - 'source-dir' => '/etc/hosts', - 'target-dir' => '/etc/hosts', - 'options' => 'ro', - }, - 'redis-localtime' => { - 'source-dir' => '/etc/localtime', - 'target-dir' => '/etc/localtime', - 'options' => 'ro', - }, - 'redis-lib' => { - 'source-dir' => '/var/lib/redis', - 'target-dir' => '/var/lib/redis', - 'options' => 'rw', - }, - 'redis-log' => { - 'source-dir' => '/var/log/redis', - 'target-dir' => '/var/log/redis', - 'options' => 'rw', - }, - 'redis-run' => { - 'source-dir' => '/var/run/redis', - 'target-dir' => '/var/run/redis', - 'options' => 'rw', - }, - 'redis-pki-extracted' => { - 'source-dir' => '/etc/pki/ca-trust/extracted', - 'target-dir' => '/etc/pki/ca-trust/extracted', - 'options' => 'ro', - }, - 'redis-pki-ca-bundle-crt' => { - 'source-dir' => '/etc/pki/tls/certs/ca-bundle.crt', - 'target-dir' => '/etc/pki/tls/certs/ca-bundle.crt', - 'options' => 'ro', - }, - 'redis-pki-ca-bundle-trust-crt' => { - 'source-dir' => '/etc/pki/tls/certs/ca-bundle.trust.crt', - 'target-dir' => '/etc/pki/tls/certs/ca-bundle.trust.crt', - 'options' => 'ro', - }, - 'redis-pki-cert' => { - 'source-dir' => '/etc/pki/tls/cert.pem', - 'target-dir' => '/etc/pki/tls/cert.pem', - 'options' => 'ro', - }, - 'redis-dev-log' => { - 'source-dir' => '/dev/log', - 'target-dir' => '/dev/log', - 'options' => 'rw', - }, - }, + storage_maps => merge($storage_maps, $storage_maps_tls), } + if length($replication_tuples)>1 { + $tunnel_map = $replication_tuples.map |$tuple| {"${tuple[0]}:${tuple[2]}"} + $tunnel_opt = " tunnel_port_map='${tunnel_map.join(';')}' tunnel_host='${tls_tunnel_local_name}'" + } else { + $tunnel_opt='' + } pacemaker::resource::ocf { 'redis': ocf_agent_name => 'heartbeat:redis', - resource_params => 'wait_last_known_master=true', + resource_params => "wait_last_known_master=true${tunnel_opt}", master_params => '', meta_params => 'notify=true ordered=true interleave=true container-attribute-target=host', op_params => 'start timeout=200s stop timeout=200s', @@ -159,8 +339,7 @@ class tripleo::profile::pacemaker::database::redis_bundle ( expression => ['redis-role eq true'], }, bundle => 'redis-bundle', - require => [Class['::redis'], - Pacemaker::Resource::Bundle['redis-bundle']], + require => [Pacemaker::Resource::Bundle['redis-bundle']], } }