From 4846c5654cbd33526e78446d03194ac301bbbaa7 Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 8 May 2018 10:11:34 +0200 Subject: [PATCH] Support use of partitions with ceph-volume Resync ceph helpers to support use of partitions and bcache devices for OSD filestore, blockstore, journal, db and wal devices. Change-Id: Iac5eefc4c3f53b1ec2f5cfc6e98a9a2168c1c0b6 Closes-Bug: 1769678 --- lib/ceph/broker.py | 7 +- lib/ceph/utils.py | 166 +++++++++++++++++++++++++++++++++------------ 2 files changed, 127 insertions(+), 46 deletions(-) diff --git a/lib/ceph/broker.py b/lib/ceph/broker.py index 8ba2e7a9..0b6d3e24 100644 --- a/lib/ceph/broker.py +++ b/lib/ceph/broker.py @@ -370,6 +370,8 @@ def handle_erasure_pool(request, service): if erasure_profile is None: erasure_profile = "default-canonical" + app_name = request.get('app-name') + # Check for missing params if pool_name is None: msg = "Missing parameter. name is required for the pool" @@ -393,7 +395,7 @@ def handle_erasure_pool(request, service): pool = ErasurePool(service=service, name=pool_name, erasure_code_profile=erasure_profile, - percent_data=weight) + percent_data=weight, app_name=app_name) # Ok make the erasure pool if not pool_exists(service=service, name=pool_name): log("Creating pool '{}' (erasure_profile={})" @@ -426,6 +428,7 @@ def handle_replicated_pool(request, service): if osds: pg_num = min(pg_num, (len(osds) * 100 // replicas)) + app_name = request.get('app-name') # Check for missing params if pool_name is None or replicas is None: msg = "Missing parameter. name and replicas are required" @@ -446,6 +449,8 @@ def handle_replicated_pool(request, service): kwargs['percent_data'] = weight if replicas: kwargs['replicas'] = replicas + if app_name: + kwargs['app_name'] = app_name pool = ReplicatedPool(service=service, name=pool_name, **kwargs) diff --git a/lib/ceph/utils.py b/lib/ceph/utils.py index 8e0e3c2b..8f72a843 100644 --- a/lib/ceph/utils.py +++ b/lib/ceph/utils.py @@ -90,6 +90,13 @@ PACKAGES = ['ceph', 'gdisk', 'ntp', 'btrfs-tools', 'python-ceph', 'radosgw', 'xfsprogs', 'python-pyudev', 'lvm2', 'parted'] +CEPH_KEY_MANAGER = 'ceph' +VAULT_KEY_MANAGER = 'vault' +KEY_MANAGERS = [ + CEPH_KEY_MANAGER, + VAULT_KEY_MANAGER, +] + LinkSpeed = { "BASE_10": 10, "BASE_100": 100, @@ -1420,10 +1427,9 @@ def get_lvs(dev): operation failed. :returns: list: List of logical volumes provided by the block device """ - pv_dev = _partition_name(dev) - if not lvm.is_lvm_physical_volume(pv_dev): + if not lvm.is_lvm_physical_volume(dev): return [] - vg_name = lvm.list_lvm_volume_group(pv_dev) + vg_name = lvm.list_lvm_volume_group(dev) return lvm.list_logical_volumes('vg_name={}'.format(vg_name)) @@ -1463,17 +1469,42 @@ def get_devices(name): def osdize(dev, osd_format, osd_journal, reformat_osd=False, - ignore_errors=False, encrypt=False, bluestore=False): + ignore_errors=False, encrypt=False, bluestore=False, + key_manager=CEPH_KEY_MANAGER): if dev.startswith('/dev'): osdize_dev(dev, osd_format, osd_journal, reformat_osd, ignore_errors, encrypt, - bluestore) + bluestore, key_manager) else: osdize_dir(dev, encrypt, bluestore) def osdize_dev(dev, osd_format, osd_journal, reformat_osd=False, - ignore_errors=False, encrypt=False, bluestore=False): + ignore_errors=False, encrypt=False, bluestore=False, + key_manager=CEPH_KEY_MANAGER): + """ + Prepare a block device for use as a Ceph OSD + + A block device will only be prepared once during the lifetime + of the calling charm unit; future executions will be skipped. + + :param: dev: Full path to block device to use + :param: osd_format: Format for OSD filesystem + :param: osd_journal: List of block devices to use for OSD journals + :param: reformat_osd: Reformat devices that are not currently in use + which have been used previously + :param: ignore_errors: Don't fail in the event of any errors during + processing + :param: encrypt: Encrypt block devices using 'key_manager' + :param: bluestore: Use bluestore native ceph block device format + :param: key_manager: Key management approach for encryption keys + :raises subprocess.CalledProcessError: in the event that any supporting + subprocess operation failed + :raises ValueError: if an invalid key_manager is provided + """ + if key_manager not in KEY_MANAGERS: + raise ValueError('Unsupported key manager: {}'.format(key_manager)) + db = kv() osd_devices = db.get('osd-devices', []) if dev in osd_devices: @@ -1510,7 +1541,8 @@ def osdize_dev(dev, osd_format, osd_journal, reformat_osd=False, cmd = _ceph_volume(dev, osd_journal, encrypt, - bluestore) + bluestore, + key_manager) else: cmd = _ceph_disk(dev, osd_format, @@ -1591,7 +1623,8 @@ def _ceph_disk(dev, osd_format, osd_journal, encrypt=False, bluestore=False): return cmd -def _ceph_volume(dev, osd_journal, encrypt=False, bluestore=False): +def _ceph_volume(dev, osd_journal, encrypt=False, bluestore=False, + key_manager=CEPH_KEY_MANAGER): """ Prepare and activate a device for usage as a Ceph OSD using ceph-volume. @@ -1602,6 +1635,7 @@ def _ceph_volume(dev, osd_journal, encrypt=False, bluestore=False): :param: osd_journal: List of block devices to use for OSD journals :param: encrypt: Use block device encryption :param: bluestore: Use bluestore storage for OSD + :param: key_manager: dm-crypt Key Manager to use :raises subprocess.CalledProcessError: in the event that any supporting LVM operation failed. :returns: list. 'ceph-volume' command and required parameters for @@ -1620,7 +1654,7 @@ def _ceph_volume(dev, osd_journal, encrypt=False, bluestore=False): cmd.append('--filestore') main_device_type = 'data' - if encrypt: + if encrypt and key_manager == CEPH_KEY_MANAGER: cmd.append('--dmcrypt') # On-disk journal volume creation @@ -1628,16 +1662,20 @@ def _ceph_volume(dev, osd_journal, encrypt=False, bluestore=False): journal_lv_type = 'journal' cmd.append('--journal') cmd.append(_allocate_logical_volume( - dev, - journal_lv_type, - osd_fsid, - size='{}M'.format(calculate_volume_size('journal'))) + dev=dev, + lv_type=journal_lv_type, + osd_fsid=osd_fsid, + size='{}M'.format(calculate_volume_size('journal')), + encrypt=encrypt, + key_manager=key_manager) ) cmd.append('--data') - cmd.append(_allocate_logical_volume(dev, - main_device_type, - osd_fsid)) + cmd.append(_allocate_logical_volume(dev=dev, + lv_type=main_device_type, + osd_fsid=osd_fsid, + encrypt=encrypt, + key_manager=key_manager)) if bluestore: for extra_volume in ('wal', 'db'): @@ -1647,11 +1685,13 @@ def _ceph_volume(dev, osd_journal, encrypt=False, bluestore=False): least_used = find_least_used_utility_device(devices, lvs=True) cmd.append(_allocate_logical_volume( - least_used, - extra_volume, - osd_fsid, + dev=least_used, + lv_type=extra_volume, + osd_fsid=osd_fsid, size='{}M'.format(calculate_volume_size(extra_volume)), - shared=True) + shared=True, + encrypt=encrypt, + key_manager=key_manager) ) elif osd_journal: @@ -1659,11 +1699,13 @@ def _ceph_volume(dev, osd_journal, encrypt=False, bluestore=False): least_used = find_least_used_utility_device(osd_journal, lvs=True) cmd.append(_allocate_logical_volume( - least_used, - 'journal', - osd_fsid, + dev=least_used, + lv_type='journal', + osd_fsid=osd_fsid, size='{}M'.format(calculate_volume_size('journal')), - shared=True) + shared=True, + encrypt=encrypt, + key_manager=key_manager) ) return cmd @@ -1682,7 +1724,6 @@ def _partition_name(dev): return '{}1'.format(dev) -# TODO(jamespage): Deal with lockbox encrypted bluestore devices. def is_active_bluestore_device(dev): """ Determine whether provided device is part of an active @@ -1691,11 +1732,10 @@ def is_active_bluestore_device(dev): :param: dev: Full path to block device to check for Bluestore usage. :returns: boolean: indicating whether device is in active use. """ - pv_dev = _partition_name(dev) - if not lvm.is_lvm_physical_volume(pv_dev): + if not lvm.is_lvm_physical_volume(dev): return False - vg_name = lvm.list_lvm_volume_group(pv_dev) + vg_name = lvm.list_lvm_volume_group(dev) lv_name = lvm.list_logical_volumes('vg_name={}'.format(vg_name))[0] block_symlinks = glob.glob('/var/lib/ceph/osd/ceph-*/block') @@ -1761,36 +1801,66 @@ def calculate_volume_size(lv_type): return int(configured_size) / _units[lv_type] -def _initialize_disk(dev): +def _luks_uuid(dev): """ - Initialize a raw block device with a single paritition - consuming 100% of the avaliable disk space. + Check to see if dev is a LUKS encrypted volume, returning the UUID + of volume if it is. + + :param: dev: path to block device to check. + :returns: str. UUID of LUKS device or None if not a LUKS device + """ + try: + cmd = ['cryptsetup', 'luksUUID', dev] + return subprocess.check_output(cmd).decode('UTF-8').strip() + except subprocess.CalledProcessError: + return None + + +def _initialize_disk(dev, dev_uuid, encrypt=False, + key_manager=CEPH_KEY_MANAGER): + """ + Initialize a raw block device consuming 100% of the avaliable + disk space. Function assumes that block device has already been wiped. :param: dev: path to block device to initialize + :param: dev_uuid: UUID to use for any dm-crypt operations + :param: encrypt: Encrypt OSD devices using dm-crypt + :param: key_manager: Key management approach for dm-crypt keys :raises: subprocess.CalledProcessError: if any parted calls fail :returns: str: Full path to new partition. """ - partition = _partition_name(dev) - if not os.path.exists(partition): + use_vaultlocker = encrypt and key_manager == VAULT_KEY_MANAGER + + if use_vaultlocker: + # NOTE(jamespage): Check to see if already initialized as a LUKS + # volume, which indicates this is a shared block + # device for journal, db or wal volumes. + luks_uuid = _luks_uuid(dev) + if luks_uuid: + return '/dev/mapper/crypt-{}'.format(luks_uuid) + + dm_crypt = '/dev/mapper/crypt-{}'.format(dev_uuid) + + if use_vaultlocker and not os.path.exists(dm_crypt): subprocess.check_call([ - 'parted', '--script', + 'vaultlocker', + 'encrypt', + '--uuid', dev_uuid, dev, - 'mklabel', - 'gpt', ]) - subprocess.check_call([ - 'parted', '--script', - dev, - 'mkpart', - 'primary', '1', '100%', - ]) - return partition + + if use_vaultlocker: + return dm_crypt + else: + return dev def _allocate_logical_volume(dev, lv_type, osd_fsid, - size=None, shared=False): + size=None, shared=False, + encrypt=False, + key_manager=CEPH_KEY_MANAGER): """ Allocate a logical volume from a block device, ensuring any required initialization and setup of PV's and VG's to support @@ -1803,13 +1873,19 @@ def _allocate_logical_volume(dev, lv_type, osd_fsid, :param: size: Size in LVM format for the device; if unset 100% of VG :param: shared: Shared volume group (journal, wal, db) + :param: encrypt: Encrypt OSD devices using dm-crypt + :param: key_manager: dm-crypt Key Manager to use :raises subprocess.CalledProcessError: in the event that any supporting LVM or parted operation fails. :returns: str: String in the format 'vg_name/lv_name'. """ lv_name = "osd-{}-{}".format(lv_type, osd_fsid) current_volumes = lvm.list_logical_volumes() - pv_dev = _initialize_disk(dev) + if shared: + dev_uuid = str(uuid.uuid4()) + else: + dev_uuid = osd_fsid + pv_dev = _initialize_disk(dev, dev_uuid, encrypt, key_manager) vg_name = None if not lvm.is_lvm_physical_volume(pv_dev):