From 692717bd4606949f6f3731a65aa9945cd2b7bf06 Mon Sep 17 00:00:00 2001 From: Alan Bishop Date: Mon, 27 Apr 2020 13:30:18 -0700 Subject: [PATCH] Fix cinder and etcd running with internal TLS enabled The LP bug referenced below describes a number of issues when cinder tries to use etcd for its distributed lock manager with internal TLS enabled. This patch resolves issues related to generating and distributing etcd's cert and key files. - The etcd cert must contain a subject alternative name (SAN) for the etcd node's internal API IP address. This is necessary because etcd wants to use IP addresses (versus host names), and this requires the IP address be listed in the TLS certificate. - The cert and key files are generated on the host, and must be available to multiple services running in their respective containers. The cert and key files need to be bind mounted, and an ACL is required so the etcd and cinder services have permission to read the files. EnableEtcdInternalTLS, a workaround introduced in [1], still defaults to False. The default value can be switched to True after tripleo switches from using novajoin to the ansible tripleo-ipa role for registering nodes with the IdM service. [1] https://review.opendev.org/#/q/Iec0d02f8f51067098dd58beb4fe57a7fd5ab5651 Closes-Bug: #1869955 Depends-On: Ifa7452ec15b81f48d7e5fb1252f20b5af1dff95c Change-Id: I798d60818b214de9266226c8409b69525a951dd5 (cherry picked from commit 2fc1290c10f77dac816d1416ddf6c137a10110b1) --- .../cinder/cinder-api-container-puppet.yaml | 15 ++--- .../cinder-common-container-puppet.yaml | 52 ++++++++++++--- .../cinder-scheduler-container-puppet.yaml | 7 +- .../cinder-volume-container-puppet.yaml | 34 ++++++++++ deployment/etcd/etcd-container-puppet.yaml | 65 ++++++++++++++++--- 5 files changed, 144 insertions(+), 29 deletions(-) diff --git a/deployment/cinder/cinder-api-container-puppet.yaml b/deployment/cinder/cinder-api-container-puppet.yaml index e84ec2a231..05b8a82a85 100644 --- a/deployment/cinder/cinder-api-container-puppet.yaml +++ b/deployment/cinder/cinder-api-container-puppet.yaml @@ -116,6 +116,9 @@ resources: RoleName: {get_param: RoleName} RoleParameters: {get_param: RoleParameters} + CinderCommon: + type: ./cinder-common-container-puppet.yaml + ApacheServiceBase: type: ../../deployment/apache/apache-baremetal-puppet.yaml properties: @@ -360,19 +363,15 @@ outputs: test: /openstack/healthcheck volumes: list_concat: - - {get_attr: [ContainersCommon, volumes]} + - {get_attr: [CinderCommon, cinder_common_volumes]} - - /var/lib/kolla/config_files/cinder_api.json:/var/lib/kolla/config_files/config.json:ro - - /var/lib/config-data/puppet-generated/cinder:/var/lib/kolla/config_files/src:ro - - /var/log/containers/cinder:/var/log/cinder:z - /var/log/containers/httpd/cinder-api:/var/log/httpd:z - if: - internal_tls_enabled - - - /etc/pki/tls/certs/httpd:/etc/pki/tls/certs/httpd:ro - - [] - - if: - - internal_tls_enabled - - - /etc/pki/tls/private/httpd:/etc/pki/tls/private/httpd:ro + - + - /etc/pki/tls/certs/httpd:/etc/pki/tls/certs/httpd:ro + - /etc/pki/tls/private/httpd:/etc/pki/tls/private/httpd:ro - [] environment: KOLLA_CONFIG_STRATEGY: COPY_ALWAYS diff --git a/deployment/cinder/cinder-common-container-puppet.yaml b/deployment/cinder/cinder-common-container-puppet.yaml index 0551a2518d..7d98e5f052 100644 --- a/deployment/cinder/cinder-common-container-puppet.yaml +++ b/deployment/cinder/cinder-common-container-puppet.yaml @@ -1,9 +1,8 @@ heat_template_version: rocky description: > - Provides the list of Docker volumes and environment to be used by the - CinderVolume and CinderBackup services. The same list is used for - HA and non-HA deployments. + Provides the list of common container volumes and environment used by + various cinder services. parameters: EndpointMap: @@ -52,10 +51,33 @@ parameters: default: false description: Whether to enable the multipath daemon type: boolean + CinderVolumeCluster: + default: '' + description: > + The cluster name used for deploying the cinder-volume service in an + active-active (A/A) configuration. This configuration requires the + Cinder backend drivers support A/A, and the cinder-volume service not + be managed by pacemaker. If these criteria are not met then the cluster + name must be left blank. + type: string + EnableInternalTLS: + type: boolean + default: false + EnableEtcdInternalTLS: + description: Controls whether etcd and the cinder-volume service use TLS + for cinder's lock manager, even when the rest of the internal + API network is using TLS. + type: boolean + default: false conditions: multipathd_enabled: {equals: [{get_param: MultipathdEnable}, true]} + cvol_active_active_tls_enabled: + and: + - not: {equals: [{get_param: CinderVolumeCluster}, '']} + - equals: [{get_param: EnableInternalTLS}, true] + - equals: [{get_param: EnableEtcdInternalTLS}, true] resources: @@ -79,6 +101,22 @@ outputs: path: /etc/ceph state: directory + cinder_common_volumes: + description: Common volumes for all cinder services + value: &cinder_common_volumes + list_concat: + - {get_attr: [ContainersCommon, volumes]} + - + - /var/lib/config-data/puppet-generated/cinder:/var/lib/kolla/config_files/src:ro + - /var/log/containers/cinder:/var/log/cinder:z + - + if: + - cvol_active_active_tls_enabled + - + - /etc/pki/tls/certs/etcd.crt:/etc/pki/tls/certs/etcd.crt:ro + - /etc/pki/tls/private/etcd.key:/etc/pki/tls/private/etcd.key:ro + - [] + cinder_volume_host_prep_tasks: description: Host prep tasks for the cinder-volume service (HA or non-HA) value: @@ -157,11 +195,10 @@ outputs: description: Volumes for the cinder-volume container (HA or non-HA) value: list_concat: - - {get_attr: [ContainersCommon, volumes]} + - *cinder_common_volumes - {get_param: CinderVolumeOptVolumes} - - /var/lib/kolla/config_files/cinder_volume.json:/var/lib/kolla/config_files/config.json:ro - - /var/lib/config-data/puppet-generated/cinder:/var/lib/kolla/config_files/src:ro - /etc/iscsi:/var/lib/kolla/config_files/src-iscsid:ro - /etc/ceph:/var/lib/kolla/config_files/src-ceph:ro - /lib/modules:/lib/modules:ro @@ -170,7 +207,6 @@ outputs: - /sys:/sys - /var/lib/cinder:/var/lib/cinder:z - /var/lib/iscsi:/var/lib/iscsi:z - - /var/log/containers/cinder:/var/log/cinder:z - if: - multipathd_enabled @@ -193,10 +229,9 @@ outputs: description: Volumes for the cinder-backup container (HA or non-HA) value: list_concat: - - {get_attr: [ContainersCommon, volumes]} + - *cinder_common_volumes - - /var/lib/kolla/config_files/cinder_backup.json:/var/lib/kolla/config_files/config.json:ro - - /var/lib/config-data/puppet-generated/cinder:/var/lib/kolla/config_files/src:ro - /etc/iscsi:/var/lib/kolla/config_files/src-iscsid:ro - /etc/ceph:/var/lib/kolla/config_files/src-ceph:ro - /dev/:/dev/ @@ -205,7 +240,6 @@ outputs: - /lib/modules:/lib/modules:ro - /var/lib/cinder:/var/lib/cinder:z - /var/lib/iscsi:/var/lib/iscsi:z - - /var/log/containers/cinder:/var/log/cinder:z - if: - multipathd_enabled diff --git a/deployment/cinder/cinder-scheduler-container-puppet.yaml b/deployment/cinder/cinder-scheduler-container-puppet.yaml index 8addf7a90b..0067e685fd 100644 --- a/deployment/cinder/cinder-scheduler-container-puppet.yaml +++ b/deployment/cinder/cinder-scheduler-container-puppet.yaml @@ -63,6 +63,9 @@ resources: RoleName: {get_param: RoleName} RoleParameters: {get_param: RoleParameters} + CinderCommon: + type: ./cinder-common-container-puppet.yaml + outputs: role_data: description: Role data for the Cinder Scheduler role. @@ -121,11 +124,9 @@ outputs: healthcheck: {get_attr: [ContainersCommon, healthcheck_rpc_port]} volumes: list_concat: - - {get_attr: [ContainersCommon, volumes]} + - {get_attr: [CinderCommon, cinder_common_volumes]} - - /var/lib/kolla/config_files/cinder_scheduler.json:/var/lib/kolla/config_files/config.json:ro - - /var/lib/config-data/puppet-generated/cinder:/var/lib/kolla/config_files/src:ro - - /var/log/containers/cinder:/var/log/cinder:z environment: KOLLA_CONFIG_STRATEGY: COPY_ALWAYS host_prep_tasks: diff --git a/deployment/cinder/cinder-volume-container-puppet.yaml b/deployment/cinder/cinder-volume-container-puppet.yaml index b3f3934dbe..f87840fbca 100644 --- a/deployment/cinder/cinder-volume-container-puppet.yaml +++ b/deployment/cinder/cinder-volume-container-puppet.yaml @@ -167,6 +167,23 @@ parameters: via the local IP for the Etcd network. If set to true, the ip on the local node will be used. If set to false, the VIP on the Etcd network will be used instead. Defaults to false. + EnableInternalTLS: + type: boolean + default: false + EnableEtcdInternalTLS: + description: Controls whether etcd and the cinder-volume service use TLS + for cinder's lock manager, even when the rest of the internal + API network is using TLS. + type: boolean + default: false + +conditions: + + cvol_active_active_tls_enabled: + and: + - not: {equals: [{get_param: CinderVolumeCluster}, '']} + - equals: [{get_param: EnableInternalTLS}, true] + - equals: [{get_param: EnableEtcdInternalTLS}, true] resources: @@ -328,6 +345,23 @@ outputs: volumes: {get_attr: [CinderCommon, cinder_volume_volumes]} environment: {get_attr: [CinderCommon, cinder_volume_environment]} host_prep_tasks: {get_attr: [CinderCommon, cinder_volume_host_prep_tasks]} + deploy_steps_tasks: + - name: ensure cinder can access etcd's tls cert and key + become: true + acl: + path: "{{ item }}" + entity: "{{ 42407 | string }}" + etype: user + permissions: r + state: present + with_items: + - /etc/pki/tls/certs/etcd.crt + - /etc/pki/tls/private/etcd.key + vars: + cvol_active_active_tls_enabled: {if: [cvol_active_active_tls_enabled, true, false]} + when: + - cvol_active_active_tls_enabled|bool + - step|int == 3 fast_forward_upgrade_tasks: - when: - step|int == 0 diff --git a/deployment/etcd/etcd-container-puppet.yaml b/deployment/etcd/etcd-container-puppet.yaml index 1dcc202a1b..e17f05b786 100644 --- a/deployment/etcd/etcd-container-puppet.yaml +++ b/deployment/etcd/etcd-container-puppet.yaml @@ -52,6 +52,15 @@ parameters: API network is using TLS. type: boolean default: false + InternalTLSCAFile: + default: '/etc/ipa/ca.crt' + type: string + description: Specifies the default CA cert to use if TLS is used for + services in the internal network. + Debug: + default: false + description: Set to True to enable debugging on all services. + type: boolean conditions: internal_tls_enabled: @@ -59,6 +68,10 @@ conditions: - {equals: [{get_param: EnableInternalTLS}, true]} - {equals: [{get_param: EnableEtcdInternalTLS}, true]} +resources: + ContainersCommon: + type: ../containers-common.yaml + outputs: role_data: description: Role data for the etcd role. @@ -74,11 +87,6 @@ outputs: "%{hiera('fqdn_$NETWORK')}" params: $NETWORK: {get_param: [ServiceNetMap, EtcdNetwork]} - # NOTE: bind IP is found in hiera replacing the network name with the local node IP - # for the given network; replacement examples (eg. for internal_api): - # internal_api -> IP - # internal_api_uri -> [IP] - # internal_api_subnet - > IP/CIDR tripleo::profile::base::etcd::bind_ip: str_replace: template: @@ -87,6 +95,7 @@ outputs: $NETWORK: {get_param: [ServiceNetMap, EtcdNetwork]} tripleo::profile::base::etcd::client_port: '2379' tripleo::profile::base::etcd::peer_port: '2380' + etcd::debug: {get_param: Debug} etcd::initial_cluster_token: {get_param: EtcdInitialClusterToken} etcd::manage_package: false tripleo::etcd::firewall_rules: @@ -112,6 +121,18 @@ outputs: template: "etcd/%{hiera('fqdn_NETWORK')}" params: NETWORK: {get_param: [ServiceNetMap, EtcdNetwork]} + dnsnames: + - str_replace: + template: "%{hiera('fqdn_NETWORK')}" + params: + NETWORK: {get_param: [ServiceNetMap, EtcdNetwork]} + - str_replace: + template: + "%{hiera('NETWORK')}" + params: + NETWORK: {get_param: [ServiceNetMap, EtcdNetwork]} + etcd::trusted_ca_file: {get_param: InternalTLSCAFile} + etcd::peer_trusted_ca_file: {get_param: InternalTLSCAFile} - # Ensure etcd and cinder-volume aren't configured to use TLS tripleo::profile::base::etcd::enable_internal_tls: false @@ -147,10 +168,19 @@ outputs: healthcheck: test: /openstack/healthcheck volumes: - - /var/lib/etcd:/var/lib/etcd - - /etc/localtime:/etc/localtime:ro - - /var/lib/kolla/config_files/etcd.json:/var/lib/kolla/config_files/config.json:ro - - /var/lib/config-data/puppet-generated/etcd:/var/lib/kolla/config_files/src:ro + list_concat: + - {get_attr: [ContainersCommon, volumes]} + - + - /var/lib/etcd:/var/lib/etcd + - /var/lib/kolla/config_files/etcd.json:/var/lib/kolla/config_files/config.json:ro + - /var/lib/config-data/puppet-generated/etcd/:/var/lib/kolla/config_files/src:ro + - + if: + - internal_tls_enabled + - + - /etc/pki/tls/certs/etcd.crt:/etc/pki/tls/certs/etcd.crt:ro + - /etc/pki/tls/private/etcd.key:/etc/pki/tls/private/etcd.key:ro + - null environment: KOLLA_CONFIG_STRATEGY: COPY_ALWAYS container_puppet_tasks: @@ -170,6 +200,23 @@ outputs: path: /var/lib/etcd state: directory setype: svirt_sandbox_file_t + deploy_steps_tasks: + - name: ensure etcd can access its tls cert and key + become: true + acl: + path: "{{ item }}" + entity: "{{ 42413 | string }}" + etype: user + permissions: r + state: present + with_items: + - /etc/pki/tls/certs/etcd.crt + - /etc/pki/tls/private/etcd.key + vars: + internal_tls_enabled: {if: [internal_tls_enabled, true, false]} + when: + - internal_tls_enabled|bool + - step|int == 2 upgrade_tasks: [] metadata_settings: if: