From 552bd0c3530bd9e2a560dbdc0204a537dbee8621 Mon Sep 17 00:00:00 2001
From: Christian Schwede <cschwede@redhat.com>
Date: Sun, 8 Oct 2017 18:27:18 +0000
Subject: [PATCH] Add Ansible playbook/workflow to rebalance Swift rings

This patch implements the Swift ring distribution and rebalance using
an Ansible playbook, callable by a Mistral workflow.

This has several benefits that are important for Swift clusters:

- Rebalancing and ring deployment is possible without executing a
  full overcloud update. This is especially important when there is a
  longer rebalance in progress and/or multiple rebalances are required

- Rebalancing can be scheduled using Mistral, for example to
  rebalance daily in a cron-like fashion

- Avoids rebalances on every node, which is risky in case one or more
  nodes fail to rebalance and leave the cluster in an inconsistent state.

- Rebalance only if it is safe, eg. a full replication run has finished
  after the last ring update

The Ansible playbook can be executed either directly using
ansible-playbook & tripleo-ansible-inventory, or by executing the
Mistral workflow.

Change-Id: I7bb1d7d4f45bee36df1c435a11bda5d4cdaae896
---
 playbooks/swift_ring_rebalance.yaml | 123 ++++++++++++++++++++++++++++
 setup.cfg                           |   1 +
 workbooks/swift_ring_rebalance.yaml |  31 +++++++
 3 files changed, 155 insertions(+)
 create mode 100644 playbooks/swift_ring_rebalance.yaml
 create mode 100644 workbooks/swift_ring_rebalance.yaml

diff --git a/playbooks/swift_ring_rebalance.yaml b/playbooks/swift_ring_rebalance.yaml
new file mode 100644
index 000000000..e6c153214
--- /dev/null
+++ b/playbooks/swift_ring_rebalance.yaml
@@ -0,0 +1,123 @@
+---
+
+- name: Update Swift rings
+  hosts: swift_storage,swift_proxy
+  gather_facts: False
+  any_errors_fatal: true
+  max_fail_percentage: 0
+  vars:
+    base_directory: "/var/lib/config-data/puppet-generated/swift/"
+    rebalance_is_safe: False
+  environment:
+    # TODO (cschwede): Remove v3 suffix once
+    # https://review.openstack.org/#/c/473740/ is included in image
+    OS_AUTH_URL: "{{ lookup('env','OS_AUTH_URL') }}v3"
+    OS_USERNAME: "{{ lookup('env','OS_USERNAME') }}"
+    OS_PROJECT_NAME: "{{ lookup('env','OS_PROJECT_NAME') }}"
+    OS_PASSWORD: "{{ lookup('env','OS_PASSWORD') }}"
+    OS_IDENTITY_API_VERSION: "{{ lookup('env','OS_IDENTITY_API_VERSION') }}"
+  tasks:
+    - name: Get reference ring checksum
+      run_once: true
+      block:
+        - name: Ensure /tmp/swift-rings directory exists
+          file: path=/tmp/swift-rings state=directory
+
+        - name: Fetch Swift rings from undercloud
+          command: swift --insecure download -o /tmp/swift-rings.tar.gz overcloud-swift-rings swift-rings.tar.gz
+
+        - name: Extract Swift rings
+          unarchive:
+            src: /tmp/swift-rings.tar.gz
+            dest: /tmp/swift-rings
+            remote_src: yes
+
+        - name: Get reference ring checksum
+          stat:
+            path: /tmp/swift-rings/etc/swift/object.ring.gz
+          register: result_reference
+
+    - name: Get file attributes of object rings
+      stat:
+        path: "{{ base_directory }}/etc/swift/object.ring.gz"
+      register: result
+
+    - name: Abort playbook run if consistency check fails
+      fail:
+        msg: "object.ring.gz does not match reference checksum"
+      when: (result.stat.exists == True) and (result_reference.stat.exists == True) and (result_reference.stat.checksum != result.stat.checksum)
+
+    - name: Deploy missing Swift rings
+      when: result.stat.exists == False
+      block:
+        - name: Fetch missing Swift rings from undercloud
+          command: swift --insecure download -o /tmp/swift-rings.tar.gz overcloud-swift-rings swift-rings.tar.gz
+
+        - name: Extract missing Swift rings
+          unarchive:
+            src: /tmp/swift-rings.tar.gz
+            dest: /{{ base_directory }}
+            remote_src: yes
+
+    - name: Get recon data
+      command: cat /var/cache/swift/object.recon
+      register: recon
+
+    - name: Check if it is safe to continue rebalancing
+      set_fact:
+        rebalance_is_safe: True
+      when: (result.stat.exists == True) and ((recon.stdout | from_json).object_replication_last | int) > ((result.stat.mtime) | int)
+
+    - name: Show warning and stop playbook run if unsafe
+      debug:
+        msg: "Rebalancing is unsafe at the moment, stopping. Please try again later"
+      when: not rebalance_is_safe
+
+    # We exit here in case there is at least one host that fails the above check
+    - meta: end_play
+      when: not rebalance_is_safe
+
+    - name: Rebalance Swift rings
+      run_once: true
+      block:
+        - name: Ensure /tmp/swift-rings directory exists
+          file: path=/tmp/swift-rings state=directory
+
+        - name: Fetch Swift rings from undercloud
+          command: swift --insecure download -o /tmp/swift-rings.tar.gz overcloud-swift-rings swift-rings.tar.gz
+
+        - name: Extract Swift rings
+          unarchive:
+            src: /tmp/swift-rings.tar.gz
+            dest: /tmp/swift-rings
+            remote_src: yes
+
+        # Can't use with_fileglob (see https://github.com/ansible/ansible/issues/17136)
+        - name: Rebalance Swift rings
+          command: swift-ring-builder /tmp/swift-rings/etc/swift/{{ item }} rebalance
+          with_items:
+            - object.builder
+            - container.builder
+            - account.builder
+          failed_when: result.rc > 1
+          register: result
+
+        - name: Create Swift ring archive
+          archive:
+            path:
+                - "/tmp/swift-rings/etc"
+            dest: /tmp/swift-rings.tar.gz
+
+        - name: Copy Swift rings to the undercloud
+          command: swift --insecure upload --object-name swift-rings.tar.gz overcloud-swift-rings /tmp/swift-rings.tar.gz
+
+    - name: Update Swift rings on all nodes
+      block:
+        - name: Fetch Swift rings from undercloud
+          command: swift --insecure download -o /tmp/swift-rings.tar.gz overcloud-swift-rings swift-rings.tar.gz
+
+        - name: Extract Swift rings
+          unarchive:
+            src: /tmp/swift-rings.tar.gz
+            dest: /{{ base_directory }}
+            remote_src: yes
diff --git a/setup.cfg b/setup.cfg
index 5de7d436b..2c8758b91 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -40,6 +40,7 @@ data_files =
     share/tripleo-common = sudoers
     share/tripleo-common/container-images = container-images/*
     share/tripleo-common/image-yaml = image-yaml/*
+    share/tripleo-common/playbooks = playbooks/*
     share/tripleo-common/workbooks = workbooks/*
     share/tripleo-common/healthcheck = healthcheck/*
 
diff --git a/workbooks/swift_ring_rebalance.yaml b/workbooks/swift_ring_rebalance.yaml
new file mode 100644
index 000000000..510e05143
--- /dev/null
+++ b/workbooks/swift_ring_rebalance.yaml
@@ -0,0 +1,31 @@
+---
+version: '2.0'
+name: tripleo.swift_ring.v1
+description: Rebalance and distribute Swift rings using Ansible
+
+
+workflows:
+  rebalance:
+    tags:
+      - tripleo-common-managed
+
+    tasks:
+      get_private_key:
+        action: tripleo.validations.get_privkey
+        on-success: deploy_rings
+
+      deploy_rings:
+        action: tripleo.ansible-playbook
+        publish:
+          output: <% task().result %>
+        input:
+          ssh_private_key: <% task(get_private_key).result %>
+          ssh_common_args: '-o StrictHostKeyChecking=no'
+          ssh_extra_args: '-o UserKnownHostsFile=/dev/null'
+          verbosity: 1
+          remote_user: heat-admin
+          become: true
+          become_user: root
+          playbook: /usr/share/tripleo-common/playbooks/swift_ring_rebalance.yaml
+          inventory: /usr/bin/tripleo-ansible-inventory
+          use_openstack_credentials: true