From f3eeb2fa15c65a1c9c8baa14fd62f4d449517b1a Mon Sep 17 00:00:00 2001 From: Michael Gugino Date: Wed, 21 Sep 2016 16:26:48 -0400 Subject: [PATCH] Deploy files for multiple ceph clusters Deploy necessary configs and keyrings for multiple ceph cluters. Specifically, the intent is to enable multiple backends for cinder that can be accessed by compute nodes. This change will allow automatic retrieval of ceph.conf and client keyrings from multiple ceph clusters. Additionally, libvirt ceph client secrets will be created to support attaching volumes to instances from multiple ceph clusters. Change-Id: Icee061b35f374955154a3dd703444b94da0117da --- defaults/main.yml | 6 + doc/source/configure-ceph.rst | 40 +++++ .../extra-ceph-clusters-00ad154ffb0589a6.yaml | 7 + tasks/ceph_all.yml | 1 - tasks/ceph_auth.yml | 50 +++++- tasks/ceph_auth_extra.yml | 77 ++++++++ tasks/ceph_auth_extra_compute.yml | 166 ++++++++++++++++++ tasks/ceph_config.yml | 30 ++-- tasks/ceph_config_extra.yml | 56 ++++++ tasks/ceph_install_apt.yml | 4 +- templates/secret.xml.j2 | 4 +- 11 files changed, 422 insertions(+), 19 deletions(-) create mode 100644 releasenotes/notes/extra-ceph-clusters-00ad154ffb0589a6.yaml create mode 100644 tasks/ceph_auth_extra.yml create mode 100644 tasks/ceph_auth_extra_compute.yml create mode 100644 tasks/ceph_config_extra.yml diff --git a/defaults/main.yml b/defaults/main.yml index 55a2b98..f646b39 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -98,3 +98,9 @@ ceph_glance_service_names: ceph_gnocchi_service_names: - gnocchi-api - gnocchi-metricd + +ceph_extra_auth_groups: "{{ ceph_extra_config_groups }}" +ceph_extra_config_groups: + - cinder_backup + - cinder_volume +ceph_extra_compute_group: nova_compute diff --git a/doc/source/configure-ceph.rst b/doc/source/configure-ceph.rst index 15eb7b0..262007a 100644 --- a/doc/source/configure-ceph.rst +++ b/doc/source/configure-ceph.rst @@ -89,6 +89,46 @@ multiple Ceph cluster backends via the ``ceph_extra_confs`` variable. These config file sources must be present on the deployment host. +Alternatively, deployers can specify more options in ``ceph_extra_confs`` +to deploy keyrings, ceph.conf files, and configure libvirt secrets. + +.. code-block:: console + + ceph_extra_confs: + - src: "/etc/openstack_deploy/ceph2.conf" + dest: "/etc/ceph/ceph2.conf" + mon_host: 192.168.1.2 + client_name: cinder2 + keyring_src: /etc/openstack_deploy/ceph2.client.cinder2.keyring + keyring_dest: /etc/ceph/ceph2.client.cinder2.keyring + secret_uuid: '{{ cinder_ceph_client_uuid2 }}' + - src: "/etc/openstack_deploy/ceph3.conf" + dest: "/etc/ceph/ceph3.conf" + mon_host: 192.168.1.3 + client_name: cinder3 + keyring_src: /etc/openstack_deploy/ceph3.client.cinder3.keyring + keyring_dest: /etc/ceph/ceph3.client.cinder3.keyring + secret_uuid: '{{ cinder_ceph_client_uuid3 }}' + +The primary aim of this feature is to deploy multiple ceph clusters as +cinder backends and enable nova/libvirt to mount block volumes from those +backends. These settings do not override the normal deployment of +ceph client and associated setup tasks. + +Deploying multiple ceph clusters as cinder backends requires the following +adjustments to each backend in ``cinder_backends`` + +.. code-block:: console + + rbd_ceph_conf: /etc/ceph/ceph2.conf + rbd_pool: cinder_volumes_2 + rbd_user: cinder2 + rbd_secret_uuid: '{{ cinder_ceph_client_uuid2 }}' + volume_backend_name: volumes2 + +The dictionary keys ``rbd_ceph_conf``, ``rbd_user``, and ``rbd_secret_uuid`` +must be unique for each ceph cluster to used as a cinder_backend. + Monitors ~~~~~~~~ diff --git a/releasenotes/notes/extra-ceph-clusters-00ad154ffb0589a6.yaml b/releasenotes/notes/extra-ceph-clusters-00ad154ffb0589a6.yaml new file mode 100644 index 0000000..33ce7d0 --- /dev/null +++ b/releasenotes/notes/extra-ceph-clusters-00ad154ffb0589a6.yaml @@ -0,0 +1,7 @@ +--- +features: + - Variable ``ceph_extra_confs`` has been expanded to support + retrieving additional ceph.conf and keyrings from multiple + ceph clusters automatically. + - Additional libvirt ceph client secrets can be defined to + support attaching volumes from different ceph clusters. diff --git a/tasks/ceph_all.yml b/tasks/ceph_all.yml index 1354339..114c7d6 100644 --- a/tasks/ceph_all.yml +++ b/tasks/ceph_all.yml @@ -60,4 +60,3 @@ static: no when: cephx | bool tags: ceph-auth - diff --git a/tasks/ceph_auth.yml b/tasks/ceph_auth.yml index a291b07..b0ff1ae 100644 --- a/tasks/ceph_auth.yml +++ b/tasks/ceph_auth.yml @@ -38,7 +38,7 @@ until: ceph_client_keyrings|success retries: 3 tags: - - ceph-auth-client-keyrings + - ceph-auth-client-keyrings - name: Create cephkeys_access_group group group: @@ -104,9 +104,12 @@ src: secret.xml.j2 dest: /tmp/nova-secret.xml mode: "0600" + with_items: + - secret_uuid: "{{ nova_ceph_client_uuid }}" + client_name: "{{ nova_ceph_client }}" when: inventory_hostname in groups.nova_compute and libvirt_nova_defined.rc is defined and libvirt_nova_defined.rc != 0 tags: - - ceph-auth-nova-libvirt-secret + - ceph-auth-nova-libvirt-secret - name: Define libvirt nova secret command: virsh secret-define --file /tmp/nova-secret.xml @@ -149,3 +152,46 @@ when: inventory_hostname in groups.nova_compute and libvirt_nova_set tags: - ceph-auth-nova-libvirt-secret + +- name: Detect correct group for extra auth + set_fact: + ceph_in_extra_auth_group: True + with_items: "{{ ceph_extra_auth_groups }}" + when: + - ceph_extra_confs is defined + - inventory_hostname in groups[item] + tags: + - ceph-auth + - ceph-auth-extra + +- include: ceph_auth_extra.yml + when: + - ceph_in_extra_auth_group is defined + - ceph_in_extra_auth_group | bool + static: no + tags: + - ceph-auth + - ceph-auth-extra + +- name: Detect extra nova uuid secret + set_fact: + ceph_extra_nova_uuid: True + with_items: "{{ ceph_extra_confs }}" + when: + - ceph_extra_confs is defined + - inventory_hostname in groups[ceph_extra_compute_group] + - item.secret_uuid is defined + tags: + - ceph-auth + - ceph-auth-extra + - ceph-auth-libvirt-extra + +- include: ceph_auth_extra_compute.yml + when: + - ceph_extra_nova_uuid is defined + - ceph_extra_nova_uuid | bool + static: no + tags: + - ceph-auth + - ceph-auth-extra + - ceph-auth-libvirt-extra diff --git a/tasks/ceph_auth_extra.yml b/tasks/ceph_auth_extra.yml new file mode 100644 index 0000000..f5d4d62 --- /dev/null +++ b/tasks/ceph_auth_extra.yml @@ -0,0 +1,77 @@ +--- +# Copyright 2016, Walmart Stores, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +- name: Create keyring files for openstack clients from extra cluster(s) + shell: ceph auth get client.{{ item.client_name }} >/dev/null && ceph auth get-or-create client.{{ item.client_name }} > /etc/ceph/ceph.client.{{ item.client_name }}.keyring.tmp + with_items: ceph_extra_confs + delegate_to: "{{ item.mon_host }}" + when: + - item.client_name is defined + - item.mon_host is defined + with_items: ceph_extra_confs + +- name: Get extra keyring files + shell: "scp {{ item.mon_host }}:/etc/ceph/ceph.client.{{ item.client_name }}.keyring.tmp {{ item.keyring_src }}" + delegate_to: localhost + with_items: "{{ ceph_extra_confs }}" + when: + - item.mon_host is defined + - item.keyring_src is defined + - item.client_name is defined + tags: + - ceph-config-create-config + - ceph-config-extra + +- name: Secure extra keyring file permissions + file: + path: "{{ item.keyring_src }}" + state: file + mode: 0600 + delegate_to: localhost + with_items: "{{ ceph_extra_confs }}" + when: + - item.keyring_src is defined + +- name: Remove temp extra keyring files + file: + path: "/etc/ceph/ceph.client.{{ item.client_name }}.keyring.tmp" + state: absent + delegate_to: "{{ item.mon_host }}" + with_items: "{{ ceph_extra_confs }}" + when: + - item.mon_host is defined + - item.keyring_src is defined + - item.client_name is defined + tags: + - ceph-config-create-config + - ceph-config-extra + +- name: Create extra keyring files + copy: + src: "{{ item.keyring_src }}" + dest: "{{ item.keyring_dest }}" + owner: root + group: "{{ cephkeys_access_group }}" + mode: 0640 + notify: + - Restart os services + with_items: "{{ ceph_extra_confs }}" + when: + - item.keyring_src is defined + - item.keyring_dest is defined + tags: + - ceph-config-create-config + - ceph-config-extra diff --git a/tasks/ceph_auth_extra_compute.yml b/tasks/ceph_auth_extra_compute.yml new file mode 100644 index 0000000..b4539e7 --- /dev/null +++ b/tasks/ceph_auth_extra_compute.yml @@ -0,0 +1,166 @@ +--- +# Copyright 2016, Walmart Stores, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +- name: Create key files for nova_compute on extra cluster(s) + shell: ceph auth get-key client.{{ item.client_name }} > /etc/ceph/ceph.client.{{ item.client_name }}.key.tmp + with_items: "{{ ceph_extra_confs }}" + delegate_to: "{{ item.mon_host }}" + when: + - item.client_name is defined + - item.mon_host is defined + +- name: Get extra key files + shell: "scp {{ item.mon_host }}:/etc/ceph/ceph.client.{{ item.client_name }}.key.tmp /tmp/{{ item.mon_host }}{{ item.client_name }}.key.tmp" + delegate_to: localhost + with_items: "{{ ceph_extra_confs }}" + when: + - item.mon_host is defined + - item.client_name is defined + tags: + - ceph-config-create-config + - ceph-config-extra + +- name: Remove temp extra key files + file: + path: "/etc/ceph/ceph.client.{{ item.client_name }}.key.tmp" + state: absent + delegate_to: "{{ item.mon_host }}" + with_items: "{{ ceph_extra_confs }}" + when: + - item.mon_host is defined + - item.keyring_src is defined + - item.client_name is defined + tags: + - ceph-config-create-config + - ceph-config-extra + +- name: Provide extra xml files to create the secrets + template: + src: secret.xml.j2 + dest: /tmp/{{ item.mon_host }}{{ item.client_name }}-secret.xml + mode: "0600" + with_items: ceph_extra_confs + when: + - item.client_name is defined + - item.mon_host is defined + - item.secret_uuid is defined + +- name: Check if extra secret(s) are defined in libvirt pt1 + shell: "virsh secret-dumpxml {{ item.secret_uuid }} 2>&1 >/dev/null && touch /tmp/{{ item.secret_uuid }}.libvirt_secret_exists" + always_run: true + failed_when: false + changed_when: false + with_items: ceph_extra_confs + when: + - item.secret_uuid is defined + tags: + - ceph-auth-nova-libvirt-secret + +- name: Check if extra secret(s) are defined in libvirt pt2 + shell: "ls /tmp | grep \\.libvirt_secret_exists | awk -F'.' '{print $1}'" + always_run: true + failed_when: false + changed_when: false + register: libvirt_secret_exists + with_items: "{{ ceph_extra_confs }}" + when: + - item.secret_uuid is defined + tags: + - ceph-auth-nova-libvirt-secret + +- name: Define libvirt nova extra secret(s) + shell: "virsh secret-define --file /tmp/{{ item.mon_host }}{{ item.client_name }}-secret.xml" + with_items: "{{ ceph_extra_confs }}" + when: + - item.client_name is defined + - item.mon_host is defined + - item.secret_uuid is defined + - item.secret_uuid not in libvirt_secret_exists.results[0].stdout_lines + notify: + - Restart os services + tags: + - ceph-auth-nova-libvirt-secret + +- name: Check if extra secret values are set in libvirt pt1 + shell: "virsh secret-get-value {{ item.secret_uuid }} 2>&1 >/dev/null && touch /tmp/{{ item.secret_uuid }}.libvirt_secret_value_exists " + always_run: true + failed_when: false + changed_when: false + register: libvirt_nova_set + with_items: ceph_extra_confs + when: + - item.secret_uuid is defined + tags: + - ceph-auth-nova-libvirt-secret + +- name: Check if extra secret values are set in libvirt pt2 + shell: "ls /tmp | grep \\.libvirt_secret_value_exists | awk -F'.' '{print $1}'" + always_run: true + failed_when: false + changed_when: false + register: libvirt_secret_value_exists + with_items: ceph_extra_confs + when: + - item.secret_uuid is defined + tags: + - ceph-auth-nova-libvirt-secret + +- name: Set extra secret value(s) in libvirt + shell: "virsh secret-set-value --secret {{ item.secret_uuid }} --base64 $(cat /tmp/{{ item.mon_host }}{{ item.client_name }}.key.tmp)" + with_items: ceph_extra_confs + when: + - item.client_name is defined + - item.mon_host is defined + - item.secret_uuid is defined + - item.secret_uuid not in libvirt_secret_value_exists.results[0].stdout_lines + notify: + - Restart os services + tags: + - ceph-auth-nova-libvirt-secret + +# Cleanup temp files + +- name: Remove libvirt nova secret detection file + file: + path: "/tmp/{{ item.secret_uuid }}.libvirt_secret_exists" + state: "absent" + with_items: ceph_extra_confs + always_run: true + ignore_errors: true + +- name: Remove libvirt nova secret value detection file + file: + path: "/tmp/{{ item.secret_uuid }}.libvirt_secret_value_exists" + state: "absent" + with_items: ceph_extra_confs + always_run: true + ignore_errors: true + +- name: Remove libvirt nova secret file + file: + path: "/tmp/{{ item.mon_host }}{{ item.client_name }}-secret.xml" + state: "absent" + with_items: ceph_extra_confs + always_run: true + ignore_errors: true + +- name: Remove libvirt key file + file: + path: "/tmp/{{ item.mon_host }}{{ item.client_name }}.key.tmp" + state: "absent" + with_items: ceph_extra_confs + always_run: true + ignore_errors: true diff --git a/tasks/ceph_config.yml b/tasks/ceph_config.yml index 482950c..96c7539 100644 --- a/tasks/ceph_config.yml +++ b/tasks/ceph_config.yml @@ -59,16 +59,22 @@ tags: - ceph-config-create-config -- name: Create extra ceph.conf files - copy: - src: "{{ item.src }}" - dest: "{{ item.dest }}" - owner: root - group: root - mode: 0644 - notify: - - Restart os services - with_items: "{{ ceph_extra_confs }}" - when: ceph_extra_confs is defined +- name: Detect correct group for extra config + set_fact: + ceph_in_extra_config_group: True + when: + - ceph_extra_confs is defined + - inventory_hostname in groups[item] + with_items: "{{ ceph_extra_config_groups }}" tags: - - ceph-config-create-config + - ceph-config + - ceph-config-extra + +- include: ceph_config_extra.yml + when: + - ceph_in_extra_config_group is defined + - ceph_in_extra_config_group | bool + static: no + tags: + - ceph-config + - ceph-config-extra diff --git a/tasks/ceph_config_extra.yml b/tasks/ceph_config_extra.yml new file mode 100644 index 0000000..8d47ae5 --- /dev/null +++ b/tasks/ceph_config_extra.yml @@ -0,0 +1,56 @@ +--- +# Copyright 2016, Walmart Stores, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +- name: Get extra ceph.conf files + shell: "scp {{ item.mon_host }}:/etc/ceph/ceph.conf {{ item.src }}" + delegate_to: localhost + with_items: "{{ ceph_extra_confs }}" + when: + - item.mon_host is defined + - item.src is defined + tags: + - ceph-config-create-config + - ceph-config-extra + +- name: Create extra ceph.conf files + copy: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + owner: root + group: root + mode: 0644 + notify: + - Restart os services + with_items: "{{ ceph_extra_confs }}" + when: + - item.src is defined + - item.dest is defined + tags: + - ceph-config-create-config + - ceph-config-extra + +- name: Add keyring section to extra ceph.conf files + ini_file: + dest: "{{ item.dest }}" + section: "client.{{ item.client_name }}" + option: keyring + value: "{{ item.keyring_dest }}" + with_items: "{{ ceph_extra_confs }}" + when: + - item.src is defined + - item.dest is defined + - item.keyring_dest is defined + - item.client_name is defined diff --git a/tasks/ceph_install_apt.yml b/tasks/ceph_install_apt.yml index ee7ea89..bc403a4 100644 --- a/tasks/ceph_install_apt.yml +++ b/tasks/ceph_install_apt.yml @@ -21,14 +21,14 @@ path: /var/cache/apt register: apt_cache_stat tags: - - ceph-apt-packages + - ceph-apt-packages - name: Update apt if needed apt: update_cache: yes when: "ansible_date_time.epoch|float - apt_cache_stat.stat.mtime > {{cache_timeout}}" tags: - - ceph-apt-packages + - ceph-apt-packages - name: Install ceph packages apt: diff --git a/templates/secret.xml.j2 b/templates/secret.xml.j2 index 3c25cd7..9243010 100644 --- a/templates/secret.xml.j2 +++ b/templates/secret.xml.j2 @@ -1,7 +1,7 @@ - {{ nova_ceph_client_uuid}} + {{ item.secret_uuid }} - client.{{ nova_ceph_client }} secret + client.{{ item.client_name }} secret