From 2fc1290c10f77dac816d1416ddf6c137a10110b1 Mon Sep 17 00:00:00 2001 From: Alan Bishop Date: Mon, 27 Apr 2020 13:30:18 -0700 Subject: [PATCH] Fix cinder and etcd running with internal TLS enabled The LP bug referenced below describes a number of issues when cinder tries to use etcd for its distributed lock manager with internal TLS enabled. This patch resolves issues related to generating and distributing etcd's cert and key files. - The etcd cert must contain a subject alternative name (SAN) for the etcd node's internal API IP address. This is necessary because etcd wants to use IP addresses (versus host names), and this requires the IP address be listed in the TLS certificate. - The cert and key files are generated on the host, and must be available to multiple services running in their respective containers. The cert and key files need to be bind mounted, and an ACL is required so the etcd and cinder services have permission to read the files. EnableEtcdInternalTLS, a workaround introduced in [1], still defaults to False. The default value can be switched to True after tripleo switches from using novajoin to the ansible tripleo-ipa role for registering nodes with the IdM service. [1] https://review.opendev.org/#/q/Iec0d02f8f51067098dd58beb4fe57a7fd5ab5651 Closes-Bug: #1869955 Depends-On: Ifa7452ec15b81f48d7e5fb1252f20b5af1dff95c Change-Id: I798d60818b214de9266226c8409b69525a951dd5 --- .../cinder/cinder-api-container-puppet.yaml | 15 ++--- .../cinder-common-container-puppet.yaml | 52 ++++++++++++--- .../cinder-scheduler-container-puppet.yaml | 7 +- .../cinder-volume-container-puppet.yaml | 34 ++++++++++ deployment/etcd/etcd-container-puppet.yaml | 65 ++++++++++++++++--- 5 files changed, 144 insertions(+), 29 deletions(-) diff --git a/deployment/cinder/cinder-api-container-puppet.yaml b/deployment/cinder/cinder-api-container-puppet.yaml index 0b901e00db..ad276e65b2 100644 --- a/deployment/cinder/cinder-api-container-puppet.yaml +++ b/deployment/cinder/cinder-api-container-puppet.yaml @@ -111,6 +111,9 @@ resources: RoleName: {get_param: RoleName} RoleParameters: {get_param: RoleParameters} + CinderCommon: + type: ./cinder-common-container-puppet.yaml + ApacheServiceBase: type: ../../deployment/apache/apache-baremetal-puppet.yaml properties: @@ -317,19 +320,15 @@ outputs: test: /openstack/healthcheck volumes: list_concat: - - {get_attr: [ContainersCommon, volumes]} + - {get_attr: [CinderCommon, cinder_common_volumes]} - - /var/lib/kolla/config_files/cinder_api.json:/var/lib/kolla/config_files/config.json:ro - - /var/lib/config-data/puppet-generated/cinder:/var/lib/kolla/config_files/src:ro - - /var/log/containers/cinder:/var/log/cinder:z - /var/log/containers/httpd/cinder-api:/var/log/httpd:z - if: - internal_tls_enabled - - - /etc/pki/tls/certs/httpd:/etc/pki/tls/certs/httpd:ro - - [] - - if: - - internal_tls_enabled - - - /etc/pki/tls/private/httpd:/etc/pki/tls/private/httpd:ro + - + - /etc/pki/tls/certs/httpd:/etc/pki/tls/certs/httpd:ro + - /etc/pki/tls/private/httpd:/etc/pki/tls/private/httpd:ro - [] environment: KOLLA_CONFIG_STRATEGY: COPY_ALWAYS diff --git a/deployment/cinder/cinder-common-container-puppet.yaml b/deployment/cinder/cinder-common-container-puppet.yaml index 738aa7903b..ac85b49f01 100644 --- a/deployment/cinder/cinder-common-container-puppet.yaml +++ b/deployment/cinder/cinder-common-container-puppet.yaml @@ -1,9 +1,8 @@ heat_template_version: rocky description: > - Provides the list of Docker volumes and environment to be used by the - CinderVolume and CinderBackup services. The same list is used for - HA and non-HA deployments. + Provides the list of common container volumes and environment used by + various cinder services. parameters: EndpointMap: @@ -52,10 +51,33 @@ parameters: default: false description: Whether to enable the multipath daemon type: boolean + CinderVolumeCluster: + default: '' + description: > + The cluster name used for deploying the cinder-volume service in an + active-active (A/A) configuration. This configuration requires the + Cinder backend drivers support A/A, and the cinder-volume service not + be managed by pacemaker. If these criteria are not met then the cluster + name must be left blank. + type: string + EnableInternalTLS: + type: boolean + default: false + EnableEtcdInternalTLS: + description: Controls whether etcd and the cinder-volume service use TLS + for cinder's lock manager, even when the rest of the internal + API network is using TLS. + type: boolean + default: false conditions: multipathd_enabled: {equals: [{get_param: MultipathdEnable}, true]} + cvol_active_active_tls_enabled: + and: + - not: {equals: [{get_param: CinderVolumeCluster}, '']} + - equals: [{get_param: EnableInternalTLS}, true] + - equals: [{get_param: EnableEtcdInternalTLS}, true] resources: @@ -79,6 +101,22 @@ outputs: path: /etc/ceph state: directory + cinder_common_volumes: + description: Common volumes for all cinder services + value: &cinder_common_volumes + list_concat: + - {get_attr: [ContainersCommon, volumes]} + - + - /var/lib/config-data/puppet-generated/cinder:/var/lib/kolla/config_files/src:ro + - /var/log/containers/cinder:/var/log/cinder:z + - + if: + - cvol_active_active_tls_enabled + - + - /etc/pki/tls/certs/etcd.crt:/etc/pki/tls/certs/etcd.crt:ro + - /etc/pki/tls/private/etcd.key:/etc/pki/tls/private/etcd.key:ro + - [] + cinder_volume_host_prep_tasks: description: Host prep tasks for the cinder-volume service (HA or non-HA) value: @@ -157,11 +195,10 @@ outputs: description: Volumes for the cinder-volume container (HA or non-HA) value: list_concat: - - {get_attr: [ContainersCommon, volumes]} + - *cinder_common_volumes - {get_param: CinderVolumeOptVolumes} - - /var/lib/kolla/config_files/cinder_volume.json:/var/lib/kolla/config_files/config.json:ro - - /var/lib/config-data/puppet-generated/cinder:/var/lib/kolla/config_files/src:ro - /etc/iscsi:/var/lib/kolla/config_files/src-iscsid:ro - /etc/ceph:/var/lib/kolla/config_files/src-ceph:ro - /lib/modules:/lib/modules:ro @@ -170,7 +207,6 @@ outputs: - /sys:/sys - /var/lib/cinder:/var/lib/cinder:z - /var/lib/iscsi:/var/lib/iscsi:z - - /var/log/containers/cinder:/var/log/cinder:z - if: - multipathd_enabled @@ -193,10 +229,9 @@ outputs: description: Volumes for the cinder-backup container (HA or non-HA) value: list_concat: - - {get_attr: [ContainersCommon, volumes]} + - *cinder_common_volumes - - /var/lib/kolla/config_files/cinder_backup.json:/var/lib/kolla/config_files/config.json:ro - - /var/lib/config-data/puppet-generated/cinder:/var/lib/kolla/config_files/src:ro - /etc/iscsi:/var/lib/kolla/config_files/src-iscsid:ro - /etc/ceph:/var/lib/kolla/config_files/src-ceph:ro - /dev/:/dev/ @@ -205,7 +240,6 @@ outputs: - /lib/modules:/lib/modules:ro - /var/lib/cinder:/var/lib/cinder:z - /var/lib/iscsi:/var/lib/iscsi:z - - /var/log/containers/cinder:/var/log/cinder:z - if: - multipathd_enabled diff --git a/deployment/cinder/cinder-scheduler-container-puppet.yaml b/deployment/cinder/cinder-scheduler-container-puppet.yaml index 73b01aea24..f16ddc620a 100644 --- a/deployment/cinder/cinder-scheduler-container-puppet.yaml +++ b/deployment/cinder/cinder-scheduler-container-puppet.yaml @@ -63,6 +63,9 @@ resources: RoleName: {get_param: RoleName} RoleParameters: {get_param: RoleParameters} + CinderCommon: + type: ./cinder-common-container-puppet.yaml + outputs: role_data: description: Role data for the Cinder Scheduler role. @@ -121,11 +124,9 @@ outputs: healthcheck: {get_attr: [ContainersCommon, healthcheck_rpc_port]} volumes: list_concat: - - {get_attr: [ContainersCommon, volumes]} + - {get_attr: [CinderCommon, cinder_common_volumes]} - - /var/lib/kolla/config_files/cinder_scheduler.json:/var/lib/kolla/config_files/config.json:ro - - /var/lib/config-data/puppet-generated/cinder:/var/lib/kolla/config_files/src:ro - - /var/log/containers/cinder:/var/log/cinder:z environment: KOLLA_CONFIG_STRATEGY: COPY_ALWAYS host_prep_tasks: diff --git a/deployment/cinder/cinder-volume-container-puppet.yaml b/deployment/cinder/cinder-volume-container-puppet.yaml index 699b952838..406dd96f71 100644 --- a/deployment/cinder/cinder-volume-container-puppet.yaml +++ b/deployment/cinder/cinder-volume-container-puppet.yaml @@ -167,6 +167,23 @@ parameters: via the local IP for the Etcd network. If set to true, the ip on the local node will be used. If set to false, the VIP on the Etcd network will be used instead. Defaults to false. + EnableInternalTLS: + type: boolean + default: false + EnableEtcdInternalTLS: + description: Controls whether etcd and the cinder-volume service use TLS + for cinder's lock manager, even when the rest of the internal + API network is using TLS. + type: boolean + default: false + +conditions: + + cvol_active_active_tls_enabled: + and: + - not: {equals: [{get_param: CinderVolumeCluster}, '']} + - equals: [{get_param: EnableInternalTLS}, true] + - equals: [{get_param: EnableEtcdInternalTLS}, true] resources: @@ -328,6 +345,23 @@ outputs: volumes: {get_attr: [CinderCommon, cinder_volume_volumes]} environment: {get_attr: [CinderCommon, cinder_volume_environment]} host_prep_tasks: {get_attr: [CinderCommon, cinder_volume_host_prep_tasks]} + deploy_steps_tasks: + - name: ensure cinder can access etcd's tls cert and key + become: true + acl: + path: "{{ item }}" + entity: "{{ 42407 | string }}" + etype: user + permissions: r + state: present + with_items: + - /etc/pki/tls/certs/etcd.crt + - /etc/pki/tls/private/etcd.key + vars: + cvol_active_active_tls_enabled: {if: [cvol_active_active_tls_enabled, true, false]} + when: + - cvol_active_active_tls_enabled|bool + - step|int == 3 fast_forward_upgrade_tasks: - when: - step|int == 0 diff --git a/deployment/etcd/etcd-container-puppet.yaml b/deployment/etcd/etcd-container-puppet.yaml index 4e094bba73..f8e4bdaf0b 100644 --- a/deployment/etcd/etcd-container-puppet.yaml +++ b/deployment/etcd/etcd-container-puppet.yaml @@ -52,6 +52,15 @@ parameters: API network is using TLS. type: boolean default: false + InternalTLSCAFile: + default: '/etc/ipa/ca.crt' + type: string + description: Specifies the default CA cert to use if TLS is used for + services in the internal network. + Debug: + default: false + description: Set to True to enable debugging on all services. + type: boolean conditions: internal_tls_enabled: @@ -59,6 +68,10 @@ conditions: - {equals: [{get_param: EnableInternalTLS}, true]} - {equals: [{get_param: EnableEtcdInternalTLS}, true]} +resources: + ContainersCommon: + type: ../containers-common.yaml + outputs: role_data: description: Role data for the etcd role. @@ -79,11 +92,6 @@ outputs: "%{hiera('fqdn_$NETWORK')}" params: $NETWORK: {get_param: [ServiceNetMap, EtcdNetwork]} - # NOTE: bind IP is found in hiera replacing the network name with the local node IP - # for the given network; replacement examples (eg. for internal_api): - # internal_api -> IP - # internal_api_uri -> [IP] - # internal_api_subnet - > IP/CIDR tripleo::profile::base::etcd::bind_ip: str_replace: template: @@ -92,6 +100,7 @@ outputs: $NETWORK: {get_param: [ServiceNetMap, EtcdNetwork]} tripleo::profile::base::etcd::client_port: '2379' tripleo::profile::base::etcd::peer_port: '2380' + etcd::debug: {get_param: Debug} etcd::initial_cluster_token: {get_param: EtcdInitialClusterToken} etcd::manage_package: false etcd::manage_service: false @@ -112,6 +121,18 @@ outputs: template: "etcd/%{hiera('fqdn_NETWORK')}" params: NETWORK: {get_param: [ServiceNetMap, EtcdNetwork]} + dnsnames: + - str_replace: + template: "%{hiera('fqdn_NETWORK')}" + params: + NETWORK: {get_param: [ServiceNetMap, EtcdNetwork]} + - str_replace: + template: + "%{hiera('NETWORK')}" + params: + NETWORK: {get_param: [ServiceNetMap, EtcdNetwork]} + etcd::trusted_ca_file: {get_param: InternalTLSCAFile} + etcd::peer_trusted_ca_file: {get_param: InternalTLSCAFile} - # Ensure etcd and cinder-volume aren't configured to use TLS tripleo::profile::base::etcd::enable_internal_tls: false @@ -147,10 +168,19 @@ outputs: healthcheck: test: /openstack/healthcheck volumes: - - /var/lib/etcd:/var/lib/etcd - - /etc/localtime:/etc/localtime:ro - - /var/lib/kolla/config_files/etcd.json:/var/lib/kolla/config_files/config.json:ro - - /var/lib/config-data/puppet-generated/etcd:/var/lib/kolla/config_files/src:ro + list_concat: + - {get_attr: [ContainersCommon, volumes]} + - + - /var/lib/etcd:/var/lib/etcd + - /var/lib/kolla/config_files/etcd.json:/var/lib/kolla/config_files/config.json:ro + - /var/lib/config-data/puppet-generated/etcd/:/var/lib/kolla/config_files/src:ro + - + if: + - internal_tls_enabled + - + - /etc/pki/tls/certs/etcd.crt:/etc/pki/tls/certs/etcd.crt:ro + - /etc/pki/tls/private/etcd.key:/etc/pki/tls/private/etcd.key:ro + - null environment: KOLLA_CONFIG_STRATEGY: COPY_ALWAYS container_puppet_tasks: @@ -170,6 +200,23 @@ outputs: path: /var/lib/etcd state: directory setype: container_file_t + deploy_steps_tasks: + - name: ensure etcd can access its tls cert and key + become: true + acl: + path: "{{ item }}" + entity: "{{ 42413 | string }}" + etype: user + permissions: r + state: present + with_items: + - /etc/pki/tls/certs/etcd.crt + - /etc/pki/tls/private/etcd.key + vars: + internal_tls_enabled: {if: [internal_tls_enabled, true, false]} + when: + - internal_tls_enabled|bool + - step|int == 2 upgrade_tasks: [] metadata_settings: if: