Merge "Add support for block device encryption"

.gitignore

@@ -8,3 +8,5 @@ tags
 *.pyc
 tests/cirros-*
 func-results.json
+.settings
+**/__pycache__

charmhelpers/contrib/openstack/templates/haproxy.cfg

@@ -6,6 +6,7 @@ global
     group haproxy
     spread-checks 0
     stats socket /var/run/haproxy/admin.sock mode 600 level admin
+    stats socket /var/run/haproxy/operator.sock mode 600 level operator
     stats timeout 2m
 
 defaults

config.yaml

@@ -40,6 +40,17 @@ options:
     description: |
       If true, charm will attempt to unmount and overwrite existing and in-use
       block-devices (WARNING).
+  ephemeral-unmount:
+    type: string
+    default:
+    description: |
+      Cloud instances provide ephermeral storage which is normally mounted
+      on /mnt.
+      .
+      Setting this option to the path of the ephemeral mountpoint will force
+      an unmount of the corresponding device so that it can be used as a swift
+      storage device. This is useful for testing purposes (cloud deployment
+      is not a typical use case).
   zone:
     default: 1
     type: int
@@ -189,3 +200,9 @@ options:
       be loaded, the charm will fail to install.
     type: boolean
     default: False
+  encrypt:
+    default: false
+    type: boolean
+    description: |
+      Encrypt block devices used by swift using dm-crypt, making use of
+      vault for encryption key management; requires a relation to vault.

hooks/block-devices-storage-attached

@@ -0,0 +1 @@
+storage.bootstrap

hooks/block-devices-storage-detached

@@ -0,0 +1 @@
+storage.bootstrap

hooks/secrets-storage-relation-broken

@@ -0,0 +1 @@
+swift_storage_hooks.py

hooks/secrets-storage-relation-changed

@@ -0,0 +1 @@
+swift_storage_hooks.py

hooks/secrets-storage-relation-departed

@@ -0,0 +1 @@
+swift_storage_hooks.py

hooks/secrets-storage-relation-joined

@@ -0,0 +1 @@
+swift_storage_hooks.py

hooks/storage.bootstrap

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+if ! dpkg -s swift > /dev/null 2>&1; then
+    juju-log "Swift not yet installed."
+    exit 0
+fi
+
+./hooks/storage.real

hooks/storage.real

@@ -0,0 +1 @@
+swift_storage_hooks.py

hooks/swift_storage_hooks.py

@@ -14,16 +14,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import base64
+import copy
+import json
 import os
 import shutil
 import sys
+import socket
+import subprocess
 import tempfile
 
 from lib.swift_storage_utils import (
     PACKAGES,
     RESTART_MAP,
     SWIFT_SVCS,
-    determine_block_devices,
     do_openstack_upgrade,
     ensure_swift_directories,
     fetch_swift_rings,
@@ -53,16 +57,20 @@ from charmhelpers.core.hookenv import (
     relations_of_type,
     status_set,
     ingress_address,
+    DEBUG,
 )
 
 from charmhelpers.fetch import (
     apt_install,
     apt_update,
+    add_source,
     filter_installed_packages
 )
 from charmhelpers.core.host import (
     add_to_updatedb_prunepath,
     rsync,
+    write_file,
+    umount,
 )
 
 from charmhelpers.core.sysctl import create as create_sysctl
@@ -81,9 +89,12 @@ from charmhelpers.contrib.network.ip import (
 from charmhelpers.contrib.network import ufw
 from charmhelpers.contrib.charmsupport import nrpe
 from charmhelpers.contrib.hardening.harden import harden
+from charmhelpers.core.unitdata import kv
 
 from distutils.dir_util import mkpath
 
+import charmhelpers.contrib.openstack.vaultlocker as vaultlocker
+
 hooks = Hooks()
 CONFIGS = register_configs()
 NAGIOS_PLUGINS = '/usr/local/lib/nagios/plugins'
@@ -173,8 +184,6 @@ def install():
     apt_update()
     apt_install(PACKAGES, fatal=True)
     initialize_ufw()
-    status_set('maintenance', 'Setting up storage')
-    setup_storage()
     ensure_swift_directories()
 
 
@@ -186,6 +195,10 @@ def config_changed():
         initialize_ufw()
     else:
         ufw.disable()
+
+    if config('ephemeral-unmount'):
+        umount(config('ephemeral-unmount'), persist=True)
+
     if config('prefer-ipv6'):
         status_set('maintenance', 'Configuring ipv6')
         assert_charm_supports_ipv6()
@@ -198,10 +211,9 @@ def config_changed():
         status_set('maintenance', 'Running openstack upgrade')
         do_openstack_upgrade(configs=CONFIGS)
 
-    setup_storage()
+    install_vaultlocker()
 
-    for rid in relation_ids('swift-storage'):
+    configure_storage()
-        swift_storage_relation_joined(rid=rid)
 
     CONFIGS.write_all()
 
@@ -216,6 +228,17 @@ def config_changed():
     add_to_updatedb_prunepath(STORAGE_MOUNT_PATH)
 
 
+def install_vaultlocker():
+    """Determine whether vaultlocker is required and install"""
+    if config('encrypt'):
+        pkgs = ['vaultlocker', 'python-hvac']
+        installed = len(filter_installed_packages(pkgs)) == 0
+        if not installed:
+            add_source('ppa:openstack-charmers/vaultlocker')
+            apt_update(fatal=True)
+            apt_install(pkgs, fatal=True)
+
+
 @hooks.hook('upgrade-charm')
 @harden()
 def upgrade_charm():
@@ -227,6 +250,10 @@ def upgrade_charm():
 
 @hooks.hook()
 def swift_storage_relation_joined(rid=None):
+    if config('encrypt') and not vaultlocker.vault_relation_complete():
+        log('Encryption configured and vault not ready, deferring',
+            level=DEBUG)
+        return
     rel_settings = {
         'zone': config('zone'),
         'object_port': config('object-server-port'),
@@ -234,7 +261,8 @@ def swift_storage_relation_joined(rid=None):
         'account_port': config('account-server-port'),
     }
 
-    devs = determine_block_devices() or []
+    db = kv()
+    devs = db.get('prepared-devices', [])
     devs = [os.path.basename(d) for d in devs]
     rel_settings['device'] = ':'.join(devs)
     # Keep a reference of devices we are adding to the ring
@@ -272,6 +300,34 @@ def swift_storage_relation_departed():
             revoke_access(removed_client, port)
 
 
+@hooks.hook('secrets-storage-relation-joined')
+def secrets_storage_joined(relation_id=None):
+    relation_set(relation_id=relation_id,
+                 secret_backend='charm-vaultlocker',
+                 isolated=True,
+                 access_address=get_relation_ip('secrets-storage'),
+                 hostname=socket.gethostname())
+
+
+@hooks.hook('secrets-storage-relation-changed')
+def secrets_storage_changed():
+    vault_ca = relation_get('vault_ca')
+    if vault_ca:
+        vault_ca = base64.decodestring(json.loads(vault_ca).encode())
+        write_file('/usr/local/share/ca-certificates/vault-ca.crt',
+                   vault_ca, perms=0o644)
+        subprocess.check_call(['update-ca-certificates', '--fresh'])
+    configure_storage()
+
+
+@hooks.hook('storage.real')
+def configure_storage():
+    setup_storage(config('encrypt'))
+
+    for rid in relation_ids('swift-storage'):
+        swift_storage_relation_joined(rid=rid)
+
+
 @hooks.hook('nrpe-external-master-relation-joined')
 @hooks.hook('nrpe-external-master-relation-changed')
 def update_nrpe_config():
@@ -318,7 +374,10 @@ def main():
         hooks.execute(sys.argv)
     except UnregisteredHookError as e:
         log('Unknown hook {} - skipping.'.format(e))
-    set_os_workload_status(CONFIGS, REQUIRED_INTERFACES,
+    required_interfaces = copy.deepcopy(REQUIRED_INTERFACES)
+    if config('encrypt'):
+        required_interfaces['vault'] = ['secrets-storage']
+    set_os_workload_status(CONFIGS, required_interfaces,
                            charm_func=assess_status)
     os_application_version_set(VERSION_PACKAGE)
 

lib/swift_storage_utils.py

@@ -4,6 +4,7 @@ import re
 import subprocess
 import shutil
 import tempfile
+import uuid
 
 from subprocess import check_call, call, CalledProcessError, check_output
 
@@ -54,6 +55,8 @@ from charmhelpers.core.hookenv import (
     relation_ids,
     iter_units_for_relation_name,
     ingress_address,
+    storage_list,
+    storage_get,
 )
 
 from charmhelpers.contrib.network import ufw
@@ -62,6 +65,7 @@ from charmhelpers.contrib.network.ip import get_host_ip
 from charmhelpers.contrib.storage.linux.utils import (
     is_block_device,
     is_device_mounted,
+    mkfs_xfs,
 )
 
 from charmhelpers.contrib.storage.linux.loopback import (
@@ -84,6 +88,10 @@ from charmhelpers.core.decorators import (
     retry_on_exception,
 )
 
+import charmhelpers.contrib.openstack.vaultlocker as vaultlocker
+
+from charmhelpers.core.unitdata import kv
+
 PACKAGES = [
     'swift', 'swift-account', 'swift-container', 'swift-object',
     'xfsprogs', 'gdisk', 'lvm2', 'python-jinja2', 'python-psutil',
@@ -162,11 +170,14 @@ def register_configs():
                      [SwiftStorageContext()])
     configs.register('/etc/rsync-juju.d/050-swift-storage.conf',
                      [RsyncContext(), SwiftStorageServerContext()])
+    # NOTE: add VaultKVContext so interface status can be assessed
     for server in ['account', 'object', 'container']:
         configs.register('/etc/swift/%s-server.conf' % server,
                          [SwiftStorageServerContext(),
                           context.BindHostContext(),
-                          context.WorkerConfigContext()]),
+                          context.WorkerConfigContext(),
+                          vaultlocker.VaultKVContext(
+                              vaultlocker.VAULTLOCKER_BACKEND)]),
     return configs
 
 
@@ -269,6 +280,12 @@ def determine_block_devices():
     else:
         bdevs = block_device.split(' ')
 
+    # List storage instances for the 'block-devices'
+    # store declared for this charm too, and add
+    # their block device paths to the list.
+    storage_ids = storage_list('block-devices')
+    bdevs.extend((storage_get('location', s) for s in storage_ids))
+
     bdevs = list(set(bdevs))
     # attempt to ensure block devices, but filter out missing devs
     _none = ['None', 'none']
@@ -279,19 +296,6 @@ def determine_block_devices():
     return valid_bdevs
 
 
-def mkfs_xfs(bdev, force=False):
-    """Format device with XFS filesystem.
-
-    By default this should fail if the device already has a filesystem on it.
-    """
-    cmd = ['mkfs.xfs']
-    if force:
-        cmd.append("-f")
-
-    cmd += ['-i', 'size=1024', bdev]
-    check_call(cmd)
-
-
 def devstore_safe_load(devstore):
     """Attempt to decode json data and return None if an error occurs while
     also printing a log.
@@ -446,21 +450,73 @@ def ensure_devs_tracked():
                 is_device_in_ring(dev, skip_rel_check=True)
 
 
-def setup_storage():
+def setup_storage(encrypt=False):
+    # Preflight check vault relation if encryption is enabled
+    vault_kv = vaultlocker.VaultKVContext(vaultlocker.VAULTLOCKER_BACKEND)
+    context = vault_kv()
+    if encrypt and not vault_kv.complete:
+        log("Encryption requested but vault relation not complete",
+            level=DEBUG)
+        return
+    elif encrypt and vault_kv.complete:
+        # NOTE: only write vaultlocker configuration once relation is complete
+        #       otherwise we run the chance of an empty configuration file
+        #       being installed on a machine with other vaultlocker based
+        #       services
+        vaultlocker.write_vaultlocker_conf(context, priority=90)
+
     # Ensure /srv/node exists just in case no disks
     # are detected and used.
     mkdir(os.path.join('/srv', 'node'),
           owner='swift', group='swift',
           perms=0o755)
     reformat = str(config('overwrite')).lower() == "true"
+
+    db = kv()
+    prepared_devices = db.get('prepared-devices', [])
+
     for dev in determine_block_devices():
+        if dev in prepared_devices:
+            log('Device {} already processed by charm,'
+                ' skipping'.format(dev))
+            continue
+
         if is_device_in_ring(os.path.basename(dev)):
             log("Device '%s' already in the ring - ignoring" % (dev))
+            # NOTE: record existing use of device dealing with
+            #       upgrades from older versions of charms without
+            #       this feature
+            prepared_devices.append(dev)
+            db.set('prepared-devices', prepared_devices)
+            db.flush()
+            continue
+
+        # NOTE: this deals with a dm-crypt'ed block device already in
+        #       use
+        if is_device_mounted(dev):
+            log("Device '{}' is already mounted, ignoring".format(dev))
             continue
 
         if reformat:
             clean_storage(dev)
 
+        loopback_device = is_mapped_loopback_device(dev)
+        options = None
+
+        if encrypt and not loopback_device:
+            dev_uuid = str(uuid.uuid4())
+            check_call(['vaultlocker', 'encrypt',
+                        '--uuid', dev_uuid,
+                        dev])
+            dev = '/dev/mapper/crypt-{}'.format(dev_uuid)
+            options = ','.join([
+                "defaults",
+                "nofail",
+                ("x-systemd.requires="
+                 "vaultlocker-decrypt@{uuid}.service".format(uuid=dev_uuid)),
+                "comment=vaultlocker",
+            ])
+
         try:
             # If not cleaned and in use, mkfs should fail.
             mkfs_xfs(dev, force=reformat)
@@ -475,8 +531,6 @@ def setup_storage():
         _mp = os.path.join('/srv', 'node', basename)
         mkdir(_mp, owner='swift', group='swift')
 
-        options = None
-        loopback_device = is_mapped_loopback_device(dev)
         mountpoint = '/srv/node/%s' % basename
         if loopback_device:
             # If an exiting fstab entry exists using the image file as the
@@ -497,6 +551,12 @@ def setup_storage():
         check_call(['chown', '-R', 'swift:swift', mountpoint])
         check_call(['chmod', '-R', '0755', mountpoint])
 
+        # NOTE: record preparation of device - this will be used when
+        #       providing block device configuration for ring builders.
+        prepared_devices.append(dev)
+        db.set('prepared-devices', prepared_devices)
+        db.flush()
+
 
 @retry_on_exception(3, base_delay=2, exc_type=CalledProcessError)
 def fetch_swift_rings(rings_url):

metadata.yaml

@@ -28,3 +28,12 @@ provides:
     scope: container
   swift-storage:
     interface: swift
+requires:
+  secrets-storage:
+    interface: vault-kv
+storage:
+  block-devices:
+    type: block
+    multiple:
+      range: 0-
+    minimum-size: 1G

templates/vaultlocker.conf.j2

@@ -0,0 +1,6 @@
+# vaultlocker configuration from swift-proxy charm
+[vault]
+url = {{ vault_url }}
+approle = {{ role_id }}
+backend = {{ secret_backend }}
+secret_id = {{ secret_id }}

test-requirements.txt

@@ -5,12 +5,12 @@ coverage>=3.6
 mock>=1.2
 flake8>=2.2.4,<=2.4.1
 os-testr>=0.4.1
-charm-tools>=2.0.0
+charm-tools>=2.0.0;python_version=='2.7'  # cheetah templates aren't availble in Python 3+
 requests==2.6.0
 # BEGIN: Amulet OpenStack Charm Helper Requirements
 # Liberty client lower constraints
 amulet>=1.14.3,<2.0
-bundletester>=0.6.1,<1.0
+bundletester>=0.6.1,<1.0;python_version=='2.7'  # cheetah templates aren't availble in Python 3+
 python-ceilometerclient>=1.5.0
 python-cinderclient>=1.4.0
 python-glanceclient>=1.1.0

tests/basic_deployment.py

@@ -96,6 +96,7 @@ class SwiftStorageBasicDeployment(OpenStackAmuletDeployment):
             'zone': '1',
             'block-device': 'vdb',
             'overwrite': 'true',
+            'ephemeral-unmount': '/mnt',
         }
         pxc_config = {
             'innodb-buffer-pool-size': '256M',

tox.ini

@@ -26,6 +26,11 @@ basepython = python3.5
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
 
+[testenv:py36]
+basepython = python3.6
+deps = -r{toxinidir}/requirements.txt
+       -r{toxinidir}/test-requirements.txt
+
 [testenv:pep8]
 basepython = python2.7
 deps = -r{toxinidir}/requirements.txt

unit_tests/test_swift_storage_relations.py

@@ -18,7 +18,7 @@ import json
 import uuid
 import tempfile
 
-from test_utils import CharmTestCase, patch_open
+from test_utils import CharmTestCase, TestKV, patch_open
 
 with patch('hooks.charmhelpers.contrib.hardening.harden.harden') as mock_dec:
     mock_dec.side_effect = (lambda *dargs, **dkwargs: lambda f:
@@ -47,7 +47,6 @@ TO_PATCH = [
     'configure_installation_source',
     'openstack_upgrade_available',
     # swift_storage_utils
-    'determine_block_devices',
     'do_openstack_upgrade',
     'ensure_swift_directories',
     'execd_preinstall',
@@ -66,6 +65,7 @@ TO_PATCH = [
     'ufw',
     'setup_ufw',
     'revoke_access',
+    'kv',
 ]
 
 
@@ -93,6 +93,8 @@ class SwiftStorageRelationsTests(CharmTestCase):
         self.config.side_effect = self.test_config.get
         self.relation_get.side_effect = self.test_relation.get
         self.get_relation_ip.return_value = '10.10.10.2'
+        self.test_kv = TestKV()
+        self.kv.return_value = self.test_kv
 
     @patch.object(hooks, 'add_ufw_gre_rule', lambda *args: None)
     def test_prunepath(self):
@@ -108,8 +110,6 @@ class SwiftStorageRelationsTests(CharmTestCase):
         )
         self.assertTrue(self.apt_update.called)
         self.apt_install.assert_called_with(PACKAGES, fatal=True)
-
-        self.assertTrue(self.setup_storage.called)
         self.assertTrue(self.execd_preinstall.called)
 
     @patch.object(hooks, 'add_ufw_gre_rule', lambda *args: None)
@@ -197,7 +197,7 @@ class SwiftStorageRelationsTests(CharmTestCase):
         kvstore = mock_kvstore.return_value
         kvstore.__enter__.return_value = kvstore
         kvstore.get.return_value = None
-        self.determine_block_devices.return_value = ['/dev/vdb']
+        self.test_kv.set('prepared-devices', ['/dev/vdb'])
 
         hooks.swift_storage_relation_joined()
 
@@ -254,8 +254,8 @@ class SwiftStorageRelationsTests(CharmTestCase):
         test_uuid = uuid.uuid4()
         test_environ = {'JUJU_ENV_UUID': test_uuid}
         mock_environ.get.side_effect = test_environ.get
-        self.determine_block_devices.return_value = ['/dev/vdb', '/dev/vdc',
+        self.test_kv.set('prepared-devices', ['/dev/vdb', '/dev/vdc',
-                                                     '/dev/vdd']
+                                              '/dev/vdd'])
         mock_local_unit.return_value = 'test/0'
         kvstore = mock_kvstore.return_value
         kvstore.__enter__.return_value = kvstore
@@ -298,8 +298,8 @@ class SwiftStorageRelationsTests(CharmTestCase):
         test_uuid = uuid.uuid4()
         test_environ = {'JUJU_ENV_UUID': test_uuid}
         mock_environ.get.side_effect = test_environ.get
-        self.determine_block_devices.return_value = ['/dev/vdb', '/dev/vdc',
+        self.test_kv.set('prepared-devices', ['/dev/vdb', '/dev/vdc',
-                                                     '/dev/vdd']
+                                              '/dev/vdd'])
         mock_local_unit.return_value = 'test/0'
         kvstore = mock_kvstore.return_value
         kvstore.__enter__.return_value = kvstore

unit_tests/test_swift_storage_utils.py

@@ -17,7 +17,7 @@ import tempfile
 from collections import namedtuple
 
 from mock import call, patch, MagicMock
-from test_utils import CharmTestCase, patch_open
+from test_utils import CharmTestCase, TestKV, patch_open
 
 import lib.swift_storage_utils as swift_utils
 
@@ -50,6 +50,8 @@ TO_PATCH = [
     'iter_units_for_relation_name',
     'ingress_address',
     'relation_ids',
+    'vaultlocker',
+    'kv',
 ]
 
 
@@ -104,11 +106,14 @@ TARGET        SOURCE   FSTYPE OPTIONS
 """
 
 
+
 class SwiftStorageUtilsTests(CharmTestCase):
 
     def setUp(self):
         super(SwiftStorageUtilsTests, self).setUp(swift_utils, TO_PATCH)
         self.config.side_effect = self.test_config.get
+        self.test_kv = TestKV()
+        self.kv.return_value = self.test_kv
 
     def test_ensure_swift_directories(self):
         with patch('os.path.isdir') as isdir:
@@ -229,18 +234,6 @@ class SwiftStorageUtilsTests(CharmTestCase):
         self.assertTrue(_find.called)
         self.assertEqual(result, [])
 
-    def test_mkfs_xfs(self):
-        swift_utils.mkfs_xfs('/dev/sdb')
-        self.check_call.assert_called_with(
-            ['mkfs.xfs', '-i', 'size=1024', '/dev/sdb']
-        )
-
-    def test_mkfs_xfs_force(self):
-        swift_utils.mkfs_xfs('/dev/sdb', force=True)
-        self.check_call.assert_called_with(
-            ['mkfs.xfs', '-f', '-i', 'size=1024', '/dev/sdb']
-        )
-
     @patch.object(swift_utils.charmhelpers.core.fstab, "Fstab")
     @patch.object(swift_utils, 'is_device_in_ring')
     @patch.object(swift_utils, 'clean_storage')
@@ -249,6 +242,7 @@ class SwiftStorageUtilsTests(CharmTestCase):
     def test_setup_storage_no_overwrite(self, determine, mkfs, clean,
                                         mock_is_device_in_ring, mock_Fstab):
         mock_is_device_in_ring.return_value = False
+        self.is_device_mounted.return_value = False
         determine.return_value = ['/dev/vdb']
         swift_utils.setup_storage()
         self.assertFalse(clean.called)
@@ -260,6 +254,8 @@ class SwiftStorageUtilsTests(CharmTestCase):
                  perms=0o755),
             call('/srv/node/vdb', group='swift', owner='swift')
         ])
+        self.assertEqual(self.test_kv.get('prepared-devices'),
+                         ['/dev/vdb'])
 
     @patch.object(swift_utils, 'is_device_in_ring')
     @patch.object(swift_utils, 'clean_storage')
@@ -270,6 +266,7 @@ class SwiftStorageUtilsTests(CharmTestCase):
         self.test_config.set('overwrite', True)
         mock_is_device_in_ring.return_value = False
         self.is_mapped_loopback_device.return_value = None
+        self.is_device_mounted.return_value = False
         determine.return_value = ['/dev/vdb']
         swift_utils.setup_storage()
         clean.assert_called_with('/dev/vdb')
@@ -288,6 +285,8 @@ class SwiftStorageUtilsTests(CharmTestCase):
                  perms=0o755),
             call('/srv/node/vdb', group='swift', owner='swift')
         ])
+        self.assertEqual(self.test_kv.get('prepared-devices'),
+                         ['/dev/vdb'])
 
     @patch.object(swift_utils, 'is_device_in_ring')
     @patch.object(swift_utils, 'determine_block_devices')
@@ -304,6 +303,71 @@ class SwiftStorageUtilsTests(CharmTestCase):
         swift_utils.setup_storage()
         self.assertEqual(self.check_call.call_count, 0)
 
+    @patch.object(swift_utils, "uuid")
+    @patch.object(swift_utils, "vaultlocker")
+    @patch.object(swift_utils.charmhelpers.core.fstab, "Fstab")
+    @patch.object(swift_utils, 'is_device_in_ring')
+    @patch.object(swift_utils, 'clean_storage')
+    @patch.object(swift_utils, 'mkfs_xfs')
+    @patch.object(swift_utils, 'determine_block_devices')
+    def test_setup_storage_encrypt(self, determine, mkfs, clean,
+                                   mock_is_device_in_ring, mock_Fstab,
+                                   mock_vaultlocker, mock_uuid):
+        mock_context = MagicMock()
+        mock_context.complete = True
+        mock_context.return_value = 'test_context'
+        mock_vaultlocker.VaultKVContext.return_value = mock_context
+        mock_uuid.uuid4.return_value = '7c3ff7c8-fd20-4dca-9be6-6f44f213d3fe'
+        mock_is_device_in_ring.return_value = False
+        self.is_device_mounted.return_value = False
+        self.is_mapped_loopback_device.return_value = None
+        determine.return_value = ['/dev/vdb']
+        swift_utils.setup_storage(encrypt=True)
+        self.assertFalse(clean.called)
+        calls = [
+            call(['vaultlocker', 'encrypt',
+                  '--uuid', '7c3ff7c8-fd20-4dca-9be6-6f44f213d3fe',
+                  '/dev/vdb']),
+            call(['chown', '-R', 'swift:swift',
+                  '/srv/node/crypt-7c3ff7c8-fd20-4dca-9be6-6f44f213d3fe']),
+            call(['chmod', '-R', '0755',
+                  '/srv/node/crypt-7c3ff7c8-fd20-4dca-9be6-6f44f213d3fe'])
+        ]
+        self.check_call.assert_has_calls(calls)
+        self.mkdir.assert_has_calls([
+            call('/srv/node', owner='swift', group='swift',
+                 perms=0o755),
+            call('/srv/node/crypt-7c3ff7c8-fd20-4dca-9be6-6f44f213d3fe',
+                 group='swift', owner='swift')
+        ])
+        self.assertEqual(self.test_kv.get('prepared-devices'),
+                         ['/dev/mapper/crypt-7c3ff7c8-fd20-4dca-9be6-6f44f213d3fe'])
+        mock_vaultlocker.write_vaultlocker_conf.assert_called_with(
+             'test_context',
+             priority=90
+        )
+
+    @patch.object(swift_utils, "uuid")
+    @patch.object(swift_utils, "vaultlocker")
+    @patch.object(swift_utils.charmhelpers.core.fstab, "Fstab")
+    @patch.object(swift_utils, 'is_device_in_ring')
+    @patch.object(swift_utils, 'clean_storage')
+    @patch.object(swift_utils, 'mkfs_xfs')
+    @patch.object(swift_utils, 'determine_block_devices')
+    def test_setup_storage_encrypt_noready(self, determine, mkfs, clean,
+                                           mock_is_device_in_ring, mock_Fstab,
+                                           mock_vaultlocker, mock_uuid):
+        mock_context = MagicMock()
+        mock_context.complete = False
+        mock_context.return_value = {}
+        mock_vaultlocker.VaultKVContext.return_value = mock_context
+        swift_utils.setup_storage(encrypt=True)
+        mock_vaultlocker.write_vaultlocker_conf.assert_not_called()
+        clean.assert_not_called()
+        self.check_call.assert_not_called()
+        self.mkdir.assert_not_called()
+        self.assertEqual(self.test_kv.get('prepared-devices'), None)
+
     def _fake_is_device_mounted(self, device):
         if device in ["/dev/sda", "/dev/vda", "/dev/cciss/c0d0"]:
             return True
@@ -373,6 +437,7 @@ class SwiftStorageUtilsTests(CharmTestCase):
         server.return_value = 'swift_server_context'
         bind_context.return_value = 'bind_host_context'
         worker_context.return_value = 'worker_context'
+        self.vaultlocker.VaultKVContext.return_value = 'vl_context'
         self.get_os_codename_package.return_value = 'grizzly'
         configs = MagicMock()
         configs.register = MagicMock()
@@ -386,13 +451,16 @@ class SwiftStorageUtilsTests(CharmTestCase):
                  ['rsync_context', 'swift_context']),
             call('/etc/swift/account-server.conf', ['swift_context',
                                                     'bind_host_context',
-                                                    'worker_context']),
+                                                    'worker_context',
+                                                    'vl_context']),
             call('/etc/swift/object-server.conf', ['swift_context',
                                                    'bind_host_context',
-                                                   'worker_context']),
+                                                   'worker_context',
+                                                   'vl_context']),
             call('/etc/swift/container-server.conf', ['swift_context',
                                                       'bind_host_context',
-                                                      'worker_context'])
+                                                      'worker_context',
+                                                      'vl_context'])
         ]
         self.assertEqual(ex, configs.register.call_args_list)
 
@@ -434,6 +502,7 @@ class SwiftStorageUtilsTests(CharmTestCase):
         mock_is_device_in_ring.return_value = False
         determine.return_value = ["/dev/loop0", ]
         self.is_mapped_loopback_device.return_value = "/srv/test.img"
+        self.is_device_mounted.return_value = False
         swift_utils.setup_storage()
         self.mount.assert_called_with(
             "/dev/loop0",
@@ -476,6 +545,7 @@ class SwiftStorageUtilsTests(CharmTestCase):
         mock_is_device_in_ring.return_value = False
         determine.return_value = ["/dev/loop0", ]
         self.is_mapped_loopback_device.return_value = "/srv/test.img"
+        self.is_device_mounted.return_value = False
         swift_utils.setup_storage()
         self.mount.assert_called_with(
             "/srv/test.img",

unit_tests/test_utils.py

@@ -117,6 +117,23 @@ class TestRelation(object):
         return None
 
 
+class TestKV(dict):
+
+    def __init__(self):
+        super(TestKV, self).__init__()
+        self.flushed = False
+        self.data = {}
+
+    def get(self, attribute, default=None):
+        return self.data.get(attribute, default)
+
+    def set(self, attribute, value):
+        self.data[attribute] = value
+
+    def flush(self):
+        self.flushed = True
+
+
 @contextmanager
 def patch_open():
     '''Patch open() to allow mocking both open() itself and the file that is