From f0042a9a7fff1dde9d0d6c942a2fcd8f3f9450f7 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Mon, 14 Aug 2017 14:54:35 +0100 Subject: [PATCH] Sync charms.ceph to get code cleanup changes Also had to fix some imports due to changes implemented as part of the cleanup. Change-Id: I64de0ad077eaaf8ca6ac0c575c4ae7f19bccf8ee --- actions/pause_resume.py | 2 +- hooks/ceph_hooks.py | 4 +- lib/__init__.py | 0 lib/ceph/__init__.py | 2157 ----------------------- lib/ceph/{ceph_broker.py => broker.py} | 115 +- lib/ceph/ceph_helpers.py | 1557 ----------------- lib/ceph/crush_utils.py | 149 ++ lib/ceph/utils.py | 2199 ++++++++++++++++++++++++ lib/setup.py | 85 - unit_tests/test_ceph_ops.py | 84 +- 10 files changed, 2487 insertions(+), 3865 deletions(-) delete mode 100644 lib/__init__.py rename lib/ceph/{ceph_broker.py => broker.py} (88%) delete mode 100644 lib/ceph/ceph_helpers.py create mode 100644 lib/ceph/crush_utils.py create mode 100644 lib/ceph/utils.py delete mode 100644 lib/setup.py diff --git a/actions/pause_resume.py b/actions/pause_resume.py index 9bd255f..2d9023c 100755 --- a/actions/pause_resume.py +++ b/actions/pause_resume.py @@ -27,7 +27,7 @@ from charmhelpers.core.hookenv import ( action_fail, ) -from ceph import get_local_osd_ids +from ceph.utils import get_local_osd_ids from ceph_hooks import assess_status from utils import ( diff --git a/hooks/ceph_hooks.py b/hooks/ceph_hooks.py index fa6f343..6064725 100755 --- a/hooks/ceph_hooks.py +++ b/hooks/ceph_hooks.py @@ -19,8 +19,8 @@ import sys import socket sys.path.append('lib') -import ceph -from ceph.ceph_broker import ( +import ceph.utils as ceph +from ceph.broker import ( process_requests ) diff --git a/lib/__init__.py b/lib/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/lib/ceph/__init__.py b/lib/ceph/__init__.py index 6aaf4fa..e69de29 100644 --- a/lib/ceph/__init__.py +++ b/lib/ceph/__init__.py @@ -1,2157 +0,0 @@ -# Copyright 2016 Canonical Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import ctypes -import collections -import json -import random -import socket -import subprocess -import time -import os -import re -import sys -import errno -import shutil -import pyudev - -from datetime import datetime - -from charmhelpers.core import hookenv -from charmhelpers.core import templating -from charmhelpers.core.host import ( - chownr, - cmp_pkgrevno, - lsb_release, - mkdir, - mounts, - owner, - service_restart, - service_start, - service_stop, - CompareHostReleases, - is_container, -) -from charmhelpers.core.hookenv import ( - cached, - config, - log, - status_set, - DEBUG, - ERROR, - WARNING, -) -from charmhelpers.fetch import ( - apt_cache, - add_source, apt_install, apt_update) -from charmhelpers.contrib.storage.linux.ceph import ( - monitor_key_set, - monitor_key_exists, - monitor_key_get, - get_mon_map, -) -from charmhelpers.contrib.storage.linux.utils import ( - is_block_device, - zap_disk, - is_device_mounted, -) -from charmhelpers.contrib.openstack.utils import ( - get_os_codename_install_source, -) - -from ceph.ceph_helpers import check_output - -CEPH_BASE_DIR = os.path.join(os.sep, 'var', 'lib', 'ceph') -OSD_BASE_DIR = os.path.join(CEPH_BASE_DIR, 'osd') -HDPARM_FILE = os.path.join(os.sep, 'etc', 'hdparm.conf') - -LEADER = 'leader' -PEON = 'peon' -QUORUM = [LEADER, PEON] - -PACKAGES = ['ceph', 'gdisk', 'ntp', 'btrfs-tools', 'python-ceph', - 'radosgw', 'xfsprogs', 'python-pyudev'] - -LinkSpeed = { - "BASE_10": 10, - "BASE_100": 100, - "BASE_1000": 1000, - "GBASE_10": 10000, - "GBASE_40": 40000, - "GBASE_100": 100000, - "UNKNOWN": None -} - -# Mapping of adapter speed to sysctl settings -NETWORK_ADAPTER_SYSCTLS = { - # 10Gb - LinkSpeed["GBASE_10"]: { - 'net.core.rmem_default': 524287, - 'net.core.wmem_default': 524287, - 'net.core.rmem_max': 524287, - 'net.core.wmem_max': 524287, - 'net.core.optmem_max': 524287, - 'net.core.netdev_max_backlog': 300000, - 'net.ipv4.tcp_rmem': '10000000 10000000 10000000', - 'net.ipv4.tcp_wmem': '10000000 10000000 10000000', - 'net.ipv4.tcp_mem': '10000000 10000000 10000000' - }, - # Mellanox 10/40Gb - LinkSpeed["GBASE_40"]: { - 'net.ipv4.tcp_timestamps': 0, - 'net.ipv4.tcp_sack': 1, - 'net.core.netdev_max_backlog': 250000, - 'net.core.rmem_max': 4194304, - 'net.core.wmem_max': 4194304, - 'net.core.rmem_default': 4194304, - 'net.core.wmem_default': 4194304, - 'net.core.optmem_max': 4194304, - 'net.ipv4.tcp_rmem': '4096 87380 4194304', - 'net.ipv4.tcp_wmem': '4096 65536 4194304', - 'net.ipv4.tcp_low_latency': 1, - 'net.ipv4.tcp_adv_win_scale': 1 - } -} - - -class Partition(object): - def __init__(self, name, number, size, start, end, sectors, uuid): - """ - A block device partition - :param name: Name of block device - :param number: Partition number - :param size: Capacity of the device - :param start: Starting block - :param end: Ending block - :param sectors: Number of blocks - :param uuid: UUID of the partition - """ - self.name = name, - self.number = number - self.size = size - self.start = start - self.end = end - self.sectors = sectors - self.uuid = uuid - - def __str__(self): - return "number: {} start: {} end: {} sectors: {} size: {} " \ - "name: {} uuid: {}".format(self.number, self.start, - self.end, - self.sectors, self.size, - self.name, self.uuid) - - def __eq__(self, other): - if isinstance(other, self.__class__): - return self.__dict__ == other.__dict__ - return False - - def __ne__(self, other): - return not self.__eq__(other) - - -def unmounted_disks(): - """List of unmounted block devices on the current host.""" - disks = [] - context = pyudev.Context() - for device in context.list_devices(DEVTYPE='disk'): - if device['SUBSYSTEM'] == 'block': - matched = False - for block_type in [u'dm', u'loop', u'ram', u'nbd']: - if block_type in device.device_node: - matched = True - if matched: - continue - disks.append(device.device_node) - log("Found disks: {}".format(disks)) - return [disk for disk in disks if not is_device_mounted(disk)] - - -def save_sysctls(sysctl_dict, save_location): - """ - Persist the sysctls to the hard drive. - :param sysctl_dict: dict - :param save_location: path to save the settings to - :raise: IOError if anything goes wrong with writing. - """ - try: - # Persist the settings for reboots - with open(save_location, "w") as fd: - for key, value in sysctl_dict.items(): - fd.write("{}={}\n".format(key, value)) - - except IOError as e: - log("Unable to persist sysctl settings to {}. Error {}".format( - save_location, e.message), level=ERROR) - raise - - -def tune_nic(network_interface): - """ - This will set optimal sysctls for the particular network adapter. - :param network_interface: string The network adapter name. - """ - speed = get_link_speed(network_interface) - if speed in NETWORK_ADAPTER_SYSCTLS: - status_set('maintenance', 'Tuning device {}'.format( - network_interface)) - sysctl_file = os.path.join( - os.sep, - 'etc', - 'sysctl.d', - '51-ceph-osd-charm-{}.conf'.format(network_interface)) - try: - log("Saving sysctl_file: {} values: {}".format( - sysctl_file, NETWORK_ADAPTER_SYSCTLS[speed]), - level=DEBUG) - save_sysctls(sysctl_dict=NETWORK_ADAPTER_SYSCTLS[speed], - save_location=sysctl_file) - except IOError as e: - log("Write to /etc/sysctl.d/51-ceph-osd-charm-{} " - "failed. {}".format(network_interface, e.message), - level=ERROR) - - try: - # Apply the settings - log("Applying sysctl settings", level=DEBUG) - check_output(["sysctl", "-p", sysctl_file]) - except subprocess.CalledProcessError as err: - log('sysctl -p {} failed with error {}'.format(sysctl_file, - err.output), - level=ERROR) - else: - log("No settings found for network adapter: {}".format( - network_interface), level=DEBUG) - - -def get_link_speed(network_interface): - """ - This will find the link speed for a given network device. Returns None - if an error occurs. - :param network_interface: string The network adapter interface. - :return: LinkSpeed - """ - speed_path = os.path.join(os.sep, 'sys', 'class', 'net', - network_interface, 'speed') - # I'm not sure where else we'd check if this doesn't exist - if not os.path.exists(speed_path): - return LinkSpeed["UNKNOWN"] - - try: - with open(speed_path, 'r') as sysfs: - nic_speed = sysfs.readlines() - - # Did we actually read anything? - if not nic_speed: - return LinkSpeed["UNKNOWN"] - - # Try to find a sysctl match for this particular speed - for name, speed in LinkSpeed.items(): - if speed == int(nic_speed[0].strip()): - return speed - # Default to UNKNOWN if we can't find a match - return LinkSpeed["UNKNOWN"] - except IOError as e: - log("Unable to open {path} because of error: {error}".format( - path=speed_path, - error=e.message), level='error') - return LinkSpeed["UNKNOWN"] - - -def persist_settings(settings_dict): - # Write all settings to /etc/hdparm.conf - """ - This will persist the hard drive settings to the /etc/hdparm.conf file - The settings_dict should be in the form of {"uuid": {"key":"value"}} - :param settings_dict: dict of settings to save - """ - if not settings_dict: - return - - try: - templating.render(source='hdparm.conf', target=HDPARM_FILE, - context=settings_dict) - except IOError as err: - log("Unable to open {path} because of error: {error}".format( - path=HDPARM_FILE, error=err.message), level=ERROR) - except Exception as e: - # The templating.render can raise a jinja2 exception if the - # template is not found. Rather than polluting the import - # space of this charm, simply catch Exception - log('Unable to render {path} due to error: {error}'.format( - path=HDPARM_FILE, error=e.message), level=ERROR) - - -def set_max_sectors_kb(dev_name, max_sectors_size): - """ - This function sets the max_sectors_kb size of a given block device. - :param dev_name: Name of the block device to query - :param max_sectors_size: int of the max_sectors_size to save - """ - max_sectors_kb_path = os.path.join('sys', 'block', dev_name, 'queue', - 'max_sectors_kb') - try: - with open(max_sectors_kb_path, 'w') as f: - f.write(max_sectors_size) - except IOError as e: - log('Failed to write max_sectors_kb to {}. Error: {}'.format( - max_sectors_kb_path, e.message), level=ERROR) - - -def get_max_sectors_kb(dev_name): - """ - This function gets the max_sectors_kb size of a given block device. - :param dev_name: Name of the block device to query - :return: int which is either the max_sectors_kb or 0 on error. - """ - max_sectors_kb_path = os.path.join('sys', 'block', dev_name, 'queue', - 'max_sectors_kb') - - # Read in what Linux has set by default - if os.path.exists(max_sectors_kb_path): - try: - with open(max_sectors_kb_path, 'r') as f: - max_sectors_kb = f.read().strip() - return int(max_sectors_kb) - except IOError as e: - log('Failed to read max_sectors_kb to {}. Error: {}'.format( - max_sectors_kb_path, e.message), level=ERROR) - # Bail. - return 0 - return 0 - - -def get_max_hw_sectors_kb(dev_name): - """ - This function gets the max_hw_sectors_kb for a given block device. - :param dev_name: Name of the block device to query - :return: int which is either the max_hw_sectors_kb or 0 on error. - """ - max_hw_sectors_kb_path = os.path.join('sys', 'block', dev_name, 'queue', - 'max_hw_sectors_kb') - # Read in what the hardware supports - if os.path.exists(max_hw_sectors_kb_path): - try: - with open(max_hw_sectors_kb_path, 'r') as f: - max_hw_sectors_kb = f.read().strip() - return int(max_hw_sectors_kb) - except IOError as e: - log('Failed to read max_hw_sectors_kb to {}. Error: {}'.format( - max_hw_sectors_kb_path, e.message), level=ERROR) - return 0 - return 0 - - -def set_hdd_read_ahead(dev_name, read_ahead_sectors=256): - """ - This function sets the hard drive read ahead. - :param dev_name: Name of the block device to set read ahead on. - :param read_ahead_sectors: int How many sectors to read ahead. - """ - try: - # Set the read ahead sectors to 256 - log('Setting read ahead to {} for device {}'.format( - read_ahead_sectors, - dev_name)) - check_output(['hdparm', - '-a{}'.format(read_ahead_sectors), - dev_name]) - except subprocess.CalledProcessError as e: - log('hdparm failed with error: {}'.format(e.output), - level=ERROR) - - -def get_block_uuid(block_dev): - """ - This queries blkid to get the uuid for a block device. - :param block_dev: Name of the block device to query. - :return: The UUID of the device or None on Error. - """ - try: - block_info = check_output( - ['blkid', '-o', 'export', block_dev]) - for tag in block_info.split('\n'): - parts = tag.split('=') - if parts[0] == 'UUID': - return parts[1] - return None - except subprocess.CalledProcessError as err: - log('get_block_uuid failed with error: {}'.format(err.output), - level=ERROR) - return None - - -def check_max_sectors(save_settings_dict, - block_dev, - uuid): - """ - Tune the max_hw_sectors if needed. - make sure that /sys/.../max_sectors_kb matches max_hw_sectors_kb or at - least 1MB for spinning disks - If the box has a RAID card with cache this could go much bigger. - :param save_settings_dict: The dict used to persist settings - :param block_dev: A block device name: Example: /dev/sda - :param uuid: The uuid of the block device - """ - dev_name = None - path_parts = os.path.split(block_dev) - if len(path_parts) == 2: - dev_name = path_parts[1] - else: - log('Unable to determine the block device name from path: {}'.format( - block_dev)) - # Play it safe and bail - return - max_sectors_kb = get_max_sectors_kb(dev_name=dev_name) - max_hw_sectors_kb = get_max_hw_sectors_kb(dev_name=dev_name) - - if max_sectors_kb < max_hw_sectors_kb: - # OK we have a situation where the hardware supports more than Linux is - # currently requesting - config_max_sectors_kb = hookenv.config('max-sectors-kb') - if config_max_sectors_kb < max_hw_sectors_kb: - # Set the max_sectors_kb to the config.yaml value if it is less - # than the max_hw_sectors_kb - log('Setting max_sectors_kb for device {} to {}'.format( - dev_name, config_max_sectors_kb)) - save_settings_dict[ - "drive_settings"][uuid][ - "read_ahead_sect"] = config_max_sectors_kb - set_max_sectors_kb(dev_name=dev_name, - max_sectors_size=config_max_sectors_kb) - else: - # Set to the max_hw_sectors_kb - log('Setting max_sectors_kb for device {} to {}'.format( - dev_name, max_hw_sectors_kb)) - save_settings_dict[ - "drive_settings"][uuid]['read_ahead_sect'] = max_hw_sectors_kb - set_max_sectors_kb(dev_name=dev_name, - max_sectors_size=max_hw_sectors_kb) - else: - log('max_sectors_kb match max_hw_sectors_kb. No change needed for ' - 'device: {}'.format(block_dev)) - - -def tune_dev(block_dev): - """ - Try to make some intelligent decisions with HDD tuning. Future work will - include optimizing SSDs. - This function will change the read ahead sectors and the max write - sectors for each block device. - :param block_dev: A block device name: Example: /dev/sda - """ - uuid = get_block_uuid(block_dev) - if uuid is None: - log('block device {} uuid is None. Unable to save to ' - 'hdparm.conf'.format(block_dev), level=DEBUG) - return - save_settings_dict = {} - log('Tuning device {}'.format(block_dev)) - status_set('maintenance', 'Tuning device {}'.format(block_dev)) - set_hdd_read_ahead(block_dev) - save_settings_dict["drive_settings"] = {} - save_settings_dict["drive_settings"][uuid] = {} - save_settings_dict["drive_settings"][uuid]['read_ahead_sect'] = 256 - - check_max_sectors(block_dev=block_dev, - save_settings_dict=save_settings_dict, - uuid=uuid) - - persist_settings(settings_dict=save_settings_dict) - status_set('maintenance', 'Finished tuning device {}'.format(block_dev)) - - -def ceph_user(): - if get_version() > 1: - return 'ceph' - else: - return "root" - - -class CrushLocation(object): - def __init__(self, - name, - identifier, - host, - rack, - row, - datacenter, - chassis, - root): - self.name = name - self.identifier = identifier - self.host = host - self.rack = rack - self.row = row - self.datacenter = datacenter - self.chassis = chassis - self.root = root - - def __str__(self): - return "name: {} id: {} host: {} rack: {} row: {} datacenter: {} " \ - "chassis :{} root: {}".format(self.name, self.identifier, - self.host, self.rack, self.row, - self.datacenter, self.chassis, - self.root) - - def __eq__(self, other): - return not self.name < other.name and not other.name < self.name - - def __ne__(self, other): - return self.name < other.name or other.name < self.name - - def __gt__(self, other): - return self.name > other.name - - def __ge__(self, other): - return not self.name < other.name - - def __le__(self, other): - return self.name < other.name - - -def get_osd_weight(osd_id): - """ - Returns the weight of the specified OSD - :return: Float :raise: ValueError if the monmap fails to parse. - Also raises CalledProcessError if our ceph command fails - """ - try: - tree = check_output( - ['ceph', 'osd', 'tree', '--format=json']) - try: - json_tree = json.loads(tree) - # Make sure children are present in the json - if not json_tree['nodes']: - return None - for device in json_tree['nodes']: - if device['type'] == 'osd' and device['name'] == osd_id: - return device['crush_weight'] - except ValueError as v: - log("Unable to parse ceph tree json: {}. Error: {}".format( - tree, v.message)) - raise - except subprocess.CalledProcessError as e: - log("ceph osd tree command failed with message: {}".format( - e.message)) - raise - - -def get_osd_tree(service): - """ - Returns the current osd map in JSON. - :return: List. :raise: ValueError if the monmap fails to parse. - Also raises CalledProcessError if our ceph command fails - """ - try: - tree = check_output( - ['ceph', '--id', service, - 'osd', 'tree', '--format=json']) - try: - json_tree = json.loads(tree) - crush_list = [] - # Make sure children are present in the json - if not json_tree['nodes']: - return None - child_ids = json_tree['nodes'][0]['children'] - for child in json_tree['nodes']: - if child['id'] in child_ids: - crush_list.append( - CrushLocation( - name=child.get('name'), - identifier=child['id'], - host=child.get('host'), - rack=child.get('rack'), - row=child.get('row'), - datacenter=child.get('datacenter'), - chassis=child.get('chassis'), - root=child.get('root') - ) - ) - return crush_list - except ValueError as v: - log("Unable to parse ceph tree json: {}. Error: {}".format( - tree, v.message)) - raise - except subprocess.CalledProcessError as e: - log("ceph osd tree command failed with message: {}".format( - e.message)) - raise - - -def _get_child_dirs(path): - """Returns a list of directory names in the specified path. - - :param path: a full path listing of the parent directory to return child - directory names - :return: list. A list of child directories under the parent directory - :raises: ValueError if the specified path does not exist or is not a - directory, - OSError if an error occurs reading the directory listing - """ - if not os.path.exists(path): - raise ValueError('Specfied path "%s" does not exist' % path) - if not os.path.isdir(path): - raise ValueError('Specified path "%s" is not a directory' % path) - - files_in_dir = [os.path.join(path, f) for f in os.listdir(path)] - return list(filter(os.path.isdir, files_in_dir)) - - -def _get_osd_num_from_dirname(dirname): - """Parses the dirname and returns the OSD id. - - Parses a string in the form of 'ceph-{osd#}' and returns the osd number - from the directory name. - - :param dirname: the directory name to return the OSD number from - :return int: the osd number the directory name corresponds to - :raises ValueError: if the osd number cannot be parsed from the provided - directory name. - """ - match = re.search('ceph-(?P\d+)', dirname) - if not match: - raise ValueError("dirname not in correct format: %s" % dirname) - - return match.group('osd_id') - - -def get_local_osd_ids(): - """ - This will list the /var/lib/ceph/osd/* directories and try - to split the ID off of the directory name and return it in - a list - - :return: list. A list of osd identifiers :raise: OSError if - something goes wrong with listing the directory. - """ - osd_ids = [] - osd_path = os.path.join(os.sep, 'var', 'lib', 'ceph', 'osd') - if os.path.exists(osd_path): - try: - dirs = os.listdir(osd_path) - for osd_dir in dirs: - osd_id = osd_dir.split('-')[1] - if _is_int(osd_id): - osd_ids.append(osd_id) - except OSError: - raise - return osd_ids - - -def get_local_mon_ids(): - """ - This will list the /var/lib/ceph/mon/* directories and try - to split the ID off of the directory name and return it in - a list - - :return: list. A list of monitor identifiers :raise: OSError if - something goes wrong with listing the directory. - """ - mon_ids = [] - mon_path = os.path.join(os.sep, 'var', 'lib', 'ceph', 'mon') - if os.path.exists(mon_path): - try: - dirs = os.listdir(mon_path) - for mon_dir in dirs: - # Basically this takes everything after ceph- as the monitor ID - match = re.search('ceph-(?P.*)', mon_dir) - if match: - mon_ids.append(match.group('mon_id')) - except OSError: - raise - return mon_ids - - -def _is_int(v): - """Return True if the object v can be turned into an integer.""" - try: - int(v) - return True - except ValueError: - return False - - -def get_version(): - """Derive Ceph release from an installed package.""" - import apt_pkg as apt - - cache = apt_cache() - package = "ceph" - try: - pkg = cache[package] - except: - # the package is unknown to the current apt cache. - e = 'Could not determine version of package with no installation ' \ - 'candidate: %s' % package - error_out(e) - - if not pkg.current_ver: - # package is known, but no version is currently installed. - e = 'Could not determine version of uninstalled package: %s' % package - error_out(e) - - vers = apt.upstream_version(pkg.current_ver.ver_str) - - # x.y match only for 20XX.X - # and ignore patch level for other packages - match = re.match('^(\d+)\.(\d+)', vers) - - if match: - vers = match.group(0) - return float(vers) - - -def error_out(msg): - log("FATAL ERROR: %s" % msg, - level=ERROR) - sys.exit(1) - - -def is_quorum(): - asok = "/var/run/ceph/ceph-mon.{}.asok".format(socket.gethostname()) - cmd = [ - "sudo", - "-u", - ceph_user(), - "ceph", - "--admin-daemon", - asok, - "mon_status" - ] - if os.path.exists(asok): - try: - result = json.loads(check_output(cmd)) - except subprocess.CalledProcessError: - return False - except ValueError: - # Non JSON response from mon_status - return False - if result['state'] in QUORUM: - return True - else: - return False - else: - return False - - -def is_leader(): - asok = "/var/run/ceph/ceph-mon.{}.asok".format(socket.gethostname()) - cmd = [ - "sudo", - "-u", - ceph_user(), - "ceph", - "--admin-daemon", - asok, - "mon_status" - ] - if os.path.exists(asok): - try: - result = json.loads(check_output(cmd)) - except subprocess.CalledProcessError: - return False - except ValueError: - # Non JSON response from mon_status - return False - if result['state'] == LEADER: - return True - else: - return False - else: - return False - - -def wait_for_quorum(): - while not is_quorum(): - log("Waiting for quorum to be reached") - time.sleep(3) - - -def add_bootstrap_hint(peer): - asok = "/var/run/ceph/ceph-mon.{}.asok".format(socket.gethostname()) - cmd = [ - "sudo", - "-u", - ceph_user(), - "ceph", - "--admin-daemon", - asok, - "add_bootstrap_peer_hint", - peer - ] - if os.path.exists(asok): - # Ignore any errors for this call - subprocess.call(cmd) - - -DISK_FORMATS = [ - 'xfs', - 'ext4', - 'btrfs' -] - -CEPH_PARTITIONS = [ - '89C57F98-2FE5-4DC0-89C1-5EC00CEFF2BE', # ceph encrypted disk in creation - '45B0969E-9B03-4F30-B4C6-5EC00CEFF106', # ceph encrypted journal - '4FBD7E29-9D25-41B8-AFD0-5EC00CEFF05D', # ceph encrypted osd data - '4FBD7E29-9D25-41B8-AFD0-062C0CEFF05D', # ceph osd data - '45B0969E-9B03-4F30-B4C6-B4B80CEFF106', # ceph osd journal - '89C57F98-2FE5-4DC0-89C1-F3AD0CEFF2BE', # ceph disk in creation -] - - -def umount(mount_point): - """ - This function unmounts a mounted directory forcibly. This will - be used for unmounting broken hard drive mounts which may hang. - If umount returns EBUSY this will lazy unmount. - :param mount_point: str. A String representing the filesystem mount point - :return: int. Returns 0 on success. errno otherwise. - """ - libc_path = ctypes.util.find_library("c") - libc = ctypes.CDLL(libc_path, use_errno=True) - - # First try to umount with MNT_FORCE - ret = libc.umount(mount_point, 1) - if ret < 0: - err = ctypes.get_errno() - if err == errno.EBUSY: - # Detach from try. IE lazy umount - ret = libc.umount(mount_point, 2) - if ret < 0: - err = ctypes.get_errno() - return err - return 0 - else: - return err - return 0 - - -def replace_osd(dead_osd_number, - dead_osd_device, - new_osd_device, - osd_format, - osd_journal, - reformat_osd=False, - ignore_errors=False): - """ - This function will automate the replacement of a failed osd disk as much - as possible. It will revoke the keys for the old osd, remove it from the - crush map and then add a new osd into the cluster. - :param dead_osd_number: The osd number found in ceph osd tree. Example: 99 - :param dead_osd_device: The physical device. Example: /dev/sda - :param osd_format: - :param osd_journal: - :param reformat_osd: - :param ignore_errors: - """ - host_mounts = mounts() - mount_point = None - for mount in host_mounts: - if mount[1] == dead_osd_device: - mount_point = mount[0] - # need to convert dev to osd number - # also need to get the mounted drive so we can tell the admin to - # replace it - try: - # Drop this osd out of the cluster. This will begin a - # rebalance operation - status_set('maintenance', 'Removing osd {}'.format(dead_osd_number)) - check_output([ - 'ceph', - '--id', - 'osd-upgrade', - 'osd', 'out', - 'osd.{}'.format(dead_osd_number)]) - - # Kill the osd process if it's not already dead - if systemd(): - service_stop('ceph-osd@{}'.format(dead_osd_number)) - else: - check_output(['stop', 'ceph-osd', 'id={}'.format( - dead_osd_number)]) - # umount if still mounted - ret = umount(mount_point) - if ret < 0: - raise RuntimeError('umount {} failed with error: {}'.format( - mount_point, os.strerror(ret))) - # Clean up the old mount point - shutil.rmtree(mount_point) - check_output([ - 'ceph', - '--id', - 'osd-upgrade', - 'osd', 'crush', 'remove', - 'osd.{}'.format(dead_osd_number)]) - # Revoke the OSDs access keys - check_output([ - 'ceph', - '--id', - 'osd-upgrade', - 'auth', 'del', - 'osd.{}'.format(dead_osd_number)]) - check_output([ - 'ceph', - '--id', - 'osd-upgrade', - 'osd', 'rm', - 'osd.{}'.format(dead_osd_number)]) - status_set('maintenance', 'Setting up replacement osd {}'.format( - new_osd_device)) - osdize(new_osd_device, - osd_format, - osd_journal, - reformat_osd, - ignore_errors) - except subprocess.CalledProcessError as e: - log('replace_osd failed with error: ' + e.output) - - -def get_partition_list(dev): - """ - Lists the partitions of a block device - :param dev: Path to a block device. ex: /dev/sda - :return: :raise: Returns a list of Partition objects. - Raises CalledProcessException if lsblk fails - """ - partitions_list = [] - try: - partitions = get_partitions(dev) - # For each line of output - for partition in partitions: - parts = partition.split() - partitions_list.append( - Partition(number=parts[0], - start=parts[1], - end=parts[2], - sectors=parts[3], - size=parts[4], - name=parts[5], - uuid=parts[6]) - ) - return partitions_list - except subprocess.CalledProcessError: - raise - - -def is_osd_disk(dev): - partitions = get_partition_list(dev) - for partition in partitions: - try: - info = check_output(['sgdisk', '-i', partition.number, dev]) - info = info.split("\n") # IGNORE:E1103 - for line in info: - for ptype in CEPH_PARTITIONS: - sig = 'Partition GUID code: {}'.format(ptype) - if line.startswith(sig): - return True - except subprocess.CalledProcessError as e: - log("sgdisk inspection of partition {} on {} failed with " - "error: {}. Skipping".format(partition.minor, dev, e.message), - level=ERROR) - return False - - -def start_osds(devices): - # Scan for ceph block devices - rescan_osd_devices() - if cmp_pkgrevno('ceph', "0.56.6") >= 0: - # Use ceph-disk activate for directory based OSD's - for dev_or_path in devices: - if os.path.exists(dev_or_path) and os.path.isdir(dev_or_path): - subprocess.check_call(['ceph-disk', 'activate', dev_or_path]) - - -def rescan_osd_devices(): - cmd = [ - 'udevadm', 'trigger', - '--subsystem-match=block', '--action=add' - ] - - subprocess.call(cmd) - - -_bootstrap_keyring = "/var/lib/ceph/bootstrap-osd/ceph.keyring" -_upgrade_keyring = "/var/lib/ceph/osd/ceph.client.osd-upgrade.keyring" - - -def is_bootstrapped(): - return os.path.exists(_bootstrap_keyring) - - -def wait_for_bootstrap(): - while not is_bootstrapped(): - time.sleep(3) - - -def import_osd_bootstrap_key(key): - if not os.path.exists(_bootstrap_keyring): - cmd = [ - "sudo", - "-u", - ceph_user(), - 'ceph-authtool', - _bootstrap_keyring, - '--create-keyring', - '--name=client.bootstrap-osd', - '--add-key={}'.format(key) - ] - subprocess.check_call(cmd) - - -def import_osd_upgrade_key(key): - if not os.path.exists(_upgrade_keyring): - cmd = [ - "sudo", - "-u", - ceph_user(), - 'ceph-authtool', - _upgrade_keyring, - '--create-keyring', - '--name=client.osd-upgrade', - '--add-key={}'.format(key) - ] - subprocess.check_call(cmd) - - -def generate_monitor_secret(): - cmd = [ - 'ceph-authtool', - '/dev/stdout', - '--name=mon.', - '--gen-key' - ] - res = check_output(cmd) - - return "{}==".format(res.split('=')[1].strip()) - -# OSD caps taken from ceph-create-keys -_osd_bootstrap_caps = { - 'mon': [ - 'allow command osd create ...', - 'allow command osd crush set ...', - r'allow command auth add * osd allow\ * mon allow\ rwx', - 'allow command mon getmap' - ] -} - -_osd_bootstrap_caps_profile = { - 'mon': [ - 'allow profile bootstrap-osd' - ] -} - - -def parse_key(raw_key): - # get-or-create appears to have different output depending - # on whether its 'get' or 'create' - # 'create' just returns the key, 'get' is more verbose and - # needs parsing - key = None - if len(raw_key.splitlines()) == 1: - key = raw_key - else: - for element in raw_key.splitlines(): - if 'key' in element: - return element.split(' = ')[1].strip() # IGNORE:E1103 - return key - - -def get_osd_bootstrap_key(): - try: - # Attempt to get/create a key using the OSD bootstrap profile first - key = get_named_key('bootstrap-osd', - _osd_bootstrap_caps_profile) - except: - # If that fails try with the older style permissions - key = get_named_key('bootstrap-osd', - _osd_bootstrap_caps) - return key - - -_radosgw_keyring = "/etc/ceph/keyring.rados.gateway" - - -def import_radosgw_key(key): - if not os.path.exists(_radosgw_keyring): - cmd = [ - "sudo", - "-u", - ceph_user(), - 'ceph-authtool', - _radosgw_keyring, - '--create-keyring', - '--name=client.radosgw.gateway', - '--add-key={}'.format(key) - ] - subprocess.check_call(cmd) - -# OSD caps taken from ceph-create-keys -_radosgw_caps = { - 'mon': ['allow rw'], - 'osd': ['allow rwx'] -} -_upgrade_caps = { - 'mon': ['allow rwx'] -} - - -def get_radosgw_key(pool_list=None): - return get_named_key(name='radosgw.gateway', - caps=_radosgw_caps, - pool_list=pool_list) - - -def get_mds_key(name): - return create_named_keyring(entity='mds', - name=name, - caps=mds_caps) - - -_mds_bootstrap_caps_profile = { - 'mon': [ - 'allow profile bootstrap-mds' - ] -} - - -def get_mds_bootstrap_key(): - return get_named_key('bootstrap-mds', - _mds_bootstrap_caps_profile) - - -_default_caps = collections.OrderedDict([ - ('mon', ['allow r']), - ('osd', ['allow rwx']), -]) - -admin_caps = collections.OrderedDict([ - ('mds', ['allow *']), - ('mon', ['allow *']), - ('osd', ['allow *']) -]) - -mds_caps = collections.OrderedDict([ - ('osd', ['allow *']), - ('mds', ['allow']), - ('mon', ['allow rwx']), -]) - -osd_upgrade_caps = collections.OrderedDict([ - ('mon', ['allow command "config-key"', - 'allow command "osd tree"', - 'allow command "config-key list"', - 'allow command "config-key put"', - 'allow command "config-key get"', - 'allow command "config-key exists"', - 'allow command "osd out"', - 'allow command "osd in"', - 'allow command "osd rm"', - 'allow command "auth del"', - ]) -]) - - -def create_named_keyring(entity, name, caps=None): - caps = caps or _default_caps - cmd = [ - "sudo", - "-u", - ceph_user(), - 'ceph', - '--name', 'mon.', - '--keyring', - '/var/lib/ceph/mon/ceph-{}/keyring'.format( - socket.gethostname() - ), - 'auth', 'get-or-create', '{entity}.{name}'.format(entity=entity, - name=name), - ] - for subsystem, subcaps in caps.items(): - cmd.extend([subsystem, '; '.join(subcaps)]) - log("Calling check_output: {}".format(cmd), level=DEBUG) - return parse_key(check_output(cmd).strip()) # IGNORE:E1103 - - -def get_upgrade_key(): - return get_named_key('upgrade-osd', _upgrade_caps) - - -def get_named_key(name, caps=None, pool_list=None): - """ - Retrieve a specific named cephx key - :param name: String Name of key to get. - :param pool_list: The list of pools to give access to - :param caps: dict of cephx capabilities - :return: Returns a cephx key - """ - try: - # Does the key already exist? - output = check_output( - [ - 'sudo', - '-u', ceph_user(), - 'ceph', - '--name', 'mon.', - '--keyring', - '/var/lib/ceph/mon/ceph-{}/keyring'.format( - socket.gethostname() - ), - 'auth', - 'get', - 'client.{}'.format(name), - ]).strip() - return parse_key(output) - except subprocess.CalledProcessError: - # Couldn't get the key, time to create it! - log("Creating new key for {}".format(name), level=DEBUG) - caps = caps or _default_caps - cmd = [ - "sudo", - "-u", - ceph_user(), - 'ceph', - '--name', 'mon.', - '--keyring', - '/var/lib/ceph/mon/ceph-{}/keyring'.format( - socket.gethostname() - ), - 'auth', 'get-or-create', 'client.{}'.format(name), - ] - # Add capabilities - for subsystem, subcaps in caps.items(): - if subsystem == 'osd': - if pool_list: - # This will output a string similar to: - # "pool=rgw pool=rbd pool=something" - pools = " ".join(['pool={0}'.format(i) for i in pool_list]) - subcaps[0] = subcaps[0] + " " + pools - cmd.extend([subsystem, '; '.join(subcaps)]) - log("Calling check_output: {}".format(cmd), level=DEBUG) - return parse_key(check_output(cmd).strip()) # IGNORE:E1103 - - -def upgrade_key_caps(key, caps): - """ Upgrade key to have capabilities caps """ - if not is_leader(): - # Not the MON leader OR not clustered - return - cmd = [ - "sudo", "-u", ceph_user(), 'ceph', 'auth', 'caps', key - ] - for subsystem, subcaps in caps.items(): - cmd.extend([subsystem, '; '.join(subcaps)]) - subprocess.check_call(cmd) - - -@cached -def systemd(): - return CompareHostReleases(lsb_release()['DISTRIB_CODENAME']) >= 'vivid' - - -def bootstrap_monitor_cluster(secret): - hostname = socket.gethostname() - path = '/var/lib/ceph/mon/ceph-{}'.format(hostname) - done = '{}/done'.format(path) - if systemd(): - init_marker = '{}/systemd'.format(path) - else: - init_marker = '{}/upstart'.format(path) - - keyring = '/var/lib/ceph/tmp/{}.mon.keyring'.format(hostname) - - if os.path.exists(done): - log('bootstrap_monitor_cluster: mon already initialized.') - else: - # Ceph >= 0.61.3 needs this for ceph-mon fs creation - mkdir('/var/run/ceph', owner=ceph_user(), - group=ceph_user(), perms=0o755) - mkdir(path, owner=ceph_user(), group=ceph_user()) - # end changes for Ceph >= 0.61.3 - try: - subprocess.check_call(['ceph-authtool', keyring, - '--create-keyring', '--name=mon.', - '--add-key={}'.format(secret), - '--cap', 'mon', 'allow *']) - - subprocess.check_call(['ceph-mon', '--mkfs', - '-i', hostname, - '--keyring', keyring]) - chownr(path, ceph_user(), ceph_user()) - with open(done, 'w'): - pass - with open(init_marker, 'w'): - pass - - if systemd(): - subprocess.check_call(['systemctl', 'enable', 'ceph-mon']) - service_restart('ceph-mon') - else: - service_restart('ceph-mon-all') - - if cmp_pkgrevno('ceph', '12.0.0') >= 0: - # NOTE(jamespage): Later ceph releases require explicit - # call to ceph-create-keys to setup the - # admin keys for the cluster; this command - # will wait for quorum in the cluster before - # returning. - cmd = ['ceph-create-keys', '--id', hostname] - subprocess.check_call(cmd) - except: - raise - finally: - os.unlink(keyring) - - -def bootstrap_manager(): - hostname = socket.gethostname() - path = '/var/lib/ceph/mgr/ceph-{}'.format(hostname) - keyring = os.path.join(path, 'keyring') - - if os.path.exists(keyring): - log('bootstrap_manager: mgr already initialized.') - else: - mkdir(path, owner=ceph_user(), group=ceph_user()) - subprocess.check_call(['ceph', 'auth', 'get-or-create', - 'mgr.{}'.format(hostname), 'mon', - 'allow profile mgr', 'osd', 'allow *', - 'mds', 'allow *', '--out-file', - keyring]) - chownr(path, ceph_user(), ceph_user()) - - unit = 'ceph-mgr@{}'.format(hostname) - subprocess.check_call(['systemctl', 'enable', unit]) - service_restart(unit) - - -def update_monfs(): - hostname = socket.gethostname() - monfs = '/var/lib/ceph/mon/ceph-{}'.format(hostname) - if systemd(): - init_marker = '{}/systemd'.format(monfs) - else: - init_marker = '{}/upstart'.format(monfs) - if os.path.exists(monfs) and not os.path.exists(init_marker): - # Mark mon as managed by upstart so that - # it gets start correctly on reboots - with open(init_marker, 'w'): - pass - - -def maybe_zap_journal(journal_dev): - if is_osd_disk(journal_dev): - log('Looks like {} is already an OSD data' - ' or journal, skipping.'.format(journal_dev)) - return - zap_disk(journal_dev) - log("Zapped journal device {}".format(journal_dev)) - - -def get_partitions(dev): - cmd = ['partx', '--raw', '--noheadings', dev] - try: - out = check_output(cmd).splitlines() - log("get partitions: {}".format(out), level=DEBUG) - return out - except subprocess.CalledProcessError as e: - log("Can't get info for {0}: {1}".format(dev, e.output)) - return [] - - -def find_least_used_journal(journal_devices): - usages = map(lambda a: (len(get_partitions(a)), a), journal_devices) - least = min(usages, key=lambda t: t[0]) - return least[1] - - -def osdize(dev, osd_format, osd_journal, reformat_osd=False, - ignore_errors=False, encrypt=False, bluestore=False): - if dev.startswith('/dev'): - osdize_dev(dev, osd_format, osd_journal, - reformat_osd, ignore_errors, encrypt, - bluestore) - else: - osdize_dir(dev, encrypt) - - -def osdize_dev(dev, osd_format, osd_journal, reformat_osd=False, - ignore_errors=False, encrypt=False, bluestore=False): - if not os.path.exists(dev): - log('Path {} does not exist - bailing'.format(dev)) - return - - if not is_block_device(dev): - log('Path {} is not a block device - bailing'.format(dev)) - return - - if is_osd_disk(dev) and not reformat_osd: - log('Looks like {} is already an' - ' OSD data or journal, skipping.'.format(dev)) - return - - if is_device_mounted(dev): - log('Looks like {} is in use, skipping.'.format(dev)) - return - - status_set('maintenance', 'Initializing device {}'.format(dev)) - cmd = ['ceph-disk', 'prepare'] - # Later versions of ceph support more options - if cmp_pkgrevno('ceph', '0.60') >= 0: - if encrypt: - cmd.append('--dmcrypt') - if cmp_pkgrevno('ceph', '0.48.3') >= 0: - if osd_format: - cmd.append('--fs-type') - cmd.append(osd_format) - - if reformat_osd: - cmd.append('--zap-disk') - - # NOTE(jamespage): enable experimental bluestore support - if cmp_pkgrevno('ceph', '10.2.0') >= 0 and bluestore: - cmd.append('--bluestore') - - cmd.append(dev) - - if osd_journal: - least_used = find_least_used_journal(osd_journal) - cmd.append(least_used) - else: - # Just provide the device - no other options - # for older versions of ceph - cmd.append(dev) - if reformat_osd: - zap_disk(dev) - - try: - log("osdize cmd: {}".format(cmd)) - subprocess.check_call(cmd) - except subprocess.CalledProcessError: - if ignore_errors: - log('Unable to initialize device: {}'.format(dev), WARNING) - else: - log('Unable to initialize device: {}'.format(dev), ERROR) - raise - - -def osdize_dir(path, encrypt=False): - if os.path.exists(os.path.join(path, 'upstart')): - log('Path {} is already configured as an OSD - bailing'.format(path)) - return - - if cmp_pkgrevno('ceph', "0.56.6") < 0: - log('Unable to use directories for OSDs with ceph < 0.56.6', - level=ERROR) - return - - mkdir(path, owner=ceph_user(), group=ceph_user(), perms=0o755) - chownr('/var/lib/ceph', ceph_user(), ceph_user()) - cmd = [ - 'sudo', '-u', ceph_user(), - 'ceph-disk', - 'prepare', - '--data-dir', - path - ] - if cmp_pkgrevno('ceph', '0.60') >= 0: - if encrypt: - cmd.append('--dmcrypt') - log("osdize dir cmd: {}".format(cmd)) - subprocess.check_call(cmd) - - -def filesystem_mounted(fs): - return subprocess.call(['grep', '-wqs', fs, '/proc/mounts']) == 0 - - -def get_running_osds(): - """Returns a list of the pids of the current running OSD daemons""" - cmd = ['pgrep', 'ceph-osd'] - try: - result = check_output(cmd) - return result.split() - except subprocess.CalledProcessError: - return [] - - -def get_cephfs(service): - """ - List the Ceph Filesystems that exist - :rtype : list. Returns a list of the ceph filesystems - :param service: The service name to run the ceph command under - """ - if get_version() < 0.86: - # This command wasn't introduced until 0.86 ceph - return [] - try: - output = check_output(["ceph", - '--id', service, - "fs", "ls"]) - if not output: - return [] - """ - Example subprocess output: - 'name: ip-172-31-23-165, metadata pool: ip-172-31-23-165_metadata, - data pools: [ip-172-31-23-165_data ]\n' - output: filesystems: ['ip-172-31-23-165'] - """ - filesystems = [] - for line in output.splitlines(): - parts = line.split(',') - for part in parts: - if "name" in part: - filesystems.append(part.split(' ')[1]) - except subprocess.CalledProcessError: - return [] - - -def wait_for_all_monitors_to_upgrade(new_version, upgrade_key): - """ - Fairly self explanatory name. This function will wait - for all monitors in the cluster to upgrade or it will - return after a timeout period has expired. - :param new_version: str of the version to watch - :param upgrade_key: the cephx key name to use - """ - done = False - start_time = time.time() - monitor_list = [] - - mon_map = get_mon_map('admin') - if mon_map['monmap']['mons']: - for mon in mon_map['monmap']['mons']: - monitor_list.append(mon['name']) - while not done: - try: - done = all(monitor_key_exists(upgrade_key, "{}_{}_{}_done".format( - "mon", mon, new_version - )) for mon in monitor_list) - current_time = time.time() - if current_time > (start_time + 10 * 60): - raise Exception - else: - # Wait 30 seconds and test again if all monitors are upgraded - time.sleep(30) - except subprocess.CalledProcessError: - raise - - -# Edge cases: -# 1. Previous node dies on upgrade, can we retry? -def roll_monitor_cluster(new_version, upgrade_key): - """ - This is tricky to get right so here's what we're going to do. - :param new_version: str of the version to upgrade to - :param upgrade_key: the cephx key name to use when upgrading - There's 2 possible cases: Either I'm first in line or not. - If I'm not first in line I'll wait a random time between 5-30 seconds - and test to see if the previous monitor is upgraded yet. - """ - log('roll_monitor_cluster called with {}'.format(new_version)) - my_name = socket.gethostname() - monitor_list = [] - mon_map = get_mon_map('admin') - if mon_map['monmap']['mons']: - for mon in mon_map['monmap']['mons']: - monitor_list.append(mon['name']) - else: - status_set('blocked', 'Unable to get monitor cluster information') - sys.exit(1) - log('monitor_list: {}'.format(monitor_list)) - - # A sorted list of osd unit names - mon_sorted_list = sorted(monitor_list) - - try: - position = mon_sorted_list.index(my_name) - log("upgrade position: {}".format(position)) - if position == 0: - # I'm first! Roll - # First set a key to inform others I'm about to roll - lock_and_roll(upgrade_key=upgrade_key, - service='mon', - my_name=my_name, - version=new_version) - else: - # Check if the previous node has finished - status_set('waiting', - 'Waiting on {} to finish upgrading'.format( - mon_sorted_list[position - 1])) - wait_on_previous_node(upgrade_key=upgrade_key, - service='mon', - previous_node=mon_sorted_list[position - 1], - version=new_version) - lock_and_roll(upgrade_key=upgrade_key, - service='mon', - my_name=my_name, - version=new_version) - except ValueError: - log("Failed to find {} in list {}.".format( - my_name, mon_sorted_list)) - status_set('blocked', 'failed to upgrade monitor') - - -def upgrade_monitor(new_version): - current_version = get_version() - status_set("maintenance", "Upgrading monitor") - log("Current ceph version is {}".format(current_version)) - log("Upgrading to: {}".format(new_version)) - - try: - add_source(config('source'), config('key')) - apt_update(fatal=True) - except subprocess.CalledProcessError as err: - log("Adding the ceph source failed with message: {}".format( - err.message)) - status_set("blocked", "Upgrade to {} failed".format(new_version)) - sys.exit(1) - try: - if systemd(): - for mon_id in get_local_mon_ids(): - service_stop('ceph-mon@{}'.format(mon_id)) - else: - service_stop('ceph-mon-all') - apt_install(packages=determine_packages(), fatal=True) - - # Ensure the files and directories under /var/lib/ceph is chowned - # properly as part of the move to the Jewel release, which moved the - # ceph daemons to running as ceph:ceph instead of root:root. - if new_version == 'jewel': - # Ensure the ownership of Ceph's directories is correct - owner = ceph_user() - chownr(path=os.path.join(os.sep, "var", "lib", "ceph"), - owner=owner, - group=owner, - follow_links=True) - - if systemd(): - for mon_id in get_local_mon_ids(): - service_start('ceph-mon@{}'.format(mon_id)) - else: - service_start('ceph-mon-all') - except subprocess.CalledProcessError as err: - log("Stopping ceph and upgrading packages failed " - "with message: {}".format(err.message)) - status_set("blocked", "Upgrade to {} failed".format(new_version)) - sys.exit(1) - - -def lock_and_roll(upgrade_key, service, my_name, version): - start_timestamp = time.time() - - log('monitor_key_set {}_{}_{}_start {}'.format( - service, - my_name, - version, - start_timestamp)) - monitor_key_set(upgrade_key, "{}_{}_{}_start".format( - service, my_name, version), start_timestamp) - log("Rolling") - - # This should be quick - if service == 'osd': - upgrade_osd(version) - elif service == 'mon': - upgrade_monitor(version) - else: - log("Unknown service {}. Unable to upgrade".format(service), - level=ERROR) - log("Done") - - stop_timestamp = time.time() - # Set a key to inform others I am finished - log('monitor_key_set {}_{}_{}_done {}'.format(service, - my_name, - version, - stop_timestamp)) - status_set('maintenance', 'Finishing upgrade') - monitor_key_set(upgrade_key, "{}_{}_{}_done".format(service, - my_name, - version), - stop_timestamp) - - -def wait_on_previous_node(upgrade_key, service, previous_node, version): - log("Previous node is: {}".format(previous_node)) - - previous_node_finished = monitor_key_exists( - upgrade_key, - "{}_{}_{}_done".format(service, previous_node, version)) - - while previous_node_finished is False: - log("{} is not finished. Waiting".format(previous_node)) - # Has this node been trying to upgrade for longer than - # 10 minutes? - # If so then move on and consider that node dead. - - # NOTE: This assumes the clusters clocks are somewhat accurate - # If the hosts clock is really far off it may cause it to skip - # the previous node even though it shouldn't. - current_timestamp = time.time() - previous_node_start_time = monitor_key_get( - upgrade_key, - "{}_{}_{}_start".format(service, previous_node, version)) - if (current_timestamp - (10 * 60)) > previous_node_start_time: - # Previous node is probably dead. Lets move on - if previous_node_start_time is not None: - log( - "Waited 10 mins on node {}. current time: {} > " - "previous node start time: {} Moving on".format( - previous_node, - (current_timestamp - (10 * 60)), - previous_node_start_time)) - return - else: - # I have to wait. Sleep a random amount of time and then - # check if I can lock,upgrade and roll. - wait_time = random.randrange(5, 30) - log('waiting for {} seconds'.format(wait_time)) - time.sleep(wait_time) - previous_node_finished = monitor_key_exists( - upgrade_key, - "{}_{}_{}_done".format(service, previous_node, version)) - - -def get_upgrade_position(osd_sorted_list, match_name): - for index, item in enumerate(osd_sorted_list): - if item.name == match_name: - return index - return None - - -# Edge cases: -# 1. Previous node dies on upgrade, can we retry? -# 2. This assumes that the osd failure domain is not set to osd. -# It rolls an entire server at a time. -def roll_osd_cluster(new_version, upgrade_key): - """ - This is tricky to get right so here's what we're going to do. - :param new_version: str of the version to upgrade to - :param upgrade_key: the cephx key name to use when upgrading - There's 2 possible cases: Either I'm first in line or not. - If I'm not first in line I'll wait a random time between 5-30 seconds - and test to see if the previous osd is upgraded yet. - - TODO: If you're not in the same failure domain it's safe to upgrade - 1. Examine all pools and adopt the most strict failure domain policy - Example: Pool 1: Failure domain = rack - Pool 2: Failure domain = host - Pool 3: Failure domain = row - - outcome: Failure domain = host - """ - log('roll_osd_cluster called with {}'.format(new_version)) - my_name = socket.gethostname() - osd_tree = get_osd_tree(service=upgrade_key) - # A sorted list of osd unit names - osd_sorted_list = sorted(osd_tree) - log("osd_sorted_list: {}".format(osd_sorted_list)) - - try: - position = get_upgrade_position(osd_sorted_list, my_name) - log("upgrade position: {}".format(position)) - if position == 0: - # I'm first! Roll - # First set a key to inform others I'm about to roll - lock_and_roll(upgrade_key=upgrade_key, - service='osd', - my_name=my_name, - version=new_version) - else: - # Check if the previous node has finished - status_set('blocked', - 'Waiting on {} to finish upgrading'.format( - osd_sorted_list[position - 1].name)) - wait_on_previous_node( - upgrade_key=upgrade_key, - service='osd', - previous_node=osd_sorted_list[position - 1].name, - version=new_version) - lock_and_roll(upgrade_key=upgrade_key, - service='osd', - my_name=my_name, - version=new_version) - except ValueError: - log("Failed to find name {} in list {}".format( - my_name, osd_sorted_list)) - status_set('blocked', 'failed to upgrade osd') - - -def upgrade_osd(new_version): - current_version = get_version() - status_set("maintenance", "Upgrading osd") - log("Current ceph version is {}".format(current_version)) - log("Upgrading to: {}".format(new_version)) - - try: - add_source(config('source'), config('key')) - apt_update(fatal=True) - except subprocess.CalledProcessError as err: - log("Adding the ceph sources failed with message: {}".format( - err.message)) - status_set("blocked", "Upgrade to {} failed".format(new_version)) - sys.exit(1) - - try: - # Upgrade the packages before restarting the daemons. - status_set('maintenance', 'Upgrading packages to %s' % new_version) - apt_install(packages=determine_packages(), fatal=True) - - # If the upgrade does not need an ownership update of any of the - # directories in the osd service directory, then simply restart - # all of the OSDs at the same time as this will be the fastest - # way to update the code on the node. - if not dirs_need_ownership_update('osd'): - log('Restarting all OSDs to load new binaries', DEBUG) - service_restart('ceph-osd-all') - return - - # Need to change the ownership of all directories which are not OSD - # directories as well. - # TODO - this should probably be moved to the general upgrade function - # and done before mon/osd. - update_owner(CEPH_BASE_DIR, recurse_dirs=False) - non_osd_dirs = filter(lambda x: not x == 'osd', - os.listdir(CEPH_BASE_DIR)) - non_osd_dirs = map(lambda x: os.path.join(CEPH_BASE_DIR, x), - non_osd_dirs) - for path in non_osd_dirs: - update_owner(path) - - # Fast service restart wasn't an option because each of the OSD - # directories need the ownership updated for all the files on - # the OSD. Walk through the OSDs one-by-one upgrading the OSD. - for osd_dir in _get_child_dirs(OSD_BASE_DIR): - try: - osd_num = _get_osd_num_from_dirname(osd_dir) - _upgrade_single_osd(osd_num, osd_dir) - except ValueError as ex: - # Directory could not be parsed - junk directory? - log('Could not parse osd directory %s: %s' % (osd_dir, ex), - WARNING) - continue - - except (subprocess.CalledProcessError, IOError) as err: - log("Stopping ceph and upgrading packages failed " - "with message: {}".format(err.message)) - status_set("blocked", "Upgrade to {} failed".format(new_version)) - sys.exit(1) - - -def _upgrade_single_osd(osd_num, osd_dir): - """Upgrades the single OSD directory. - - :param osd_num: the num of the OSD - :param osd_dir: the directory of the OSD to upgrade - :raises CalledProcessError: if an error occurs in a command issued as part - of the upgrade process - :raises IOError: if an error occurs reading/writing to a file as part - of the upgrade process - """ - stop_osd(osd_num) - disable_osd(osd_num) - update_owner(osd_dir) - enable_osd(osd_num) - start_osd(osd_num) - - -def stop_osd(osd_num): - """Stops the specified OSD number. - - :param osd_num: the osd number to stop - """ - if systemd(): - service_stop('ceph-osd@{}'.format(osd_num)) - else: - service_stop('ceph-osd', id=osd_num) - - -def start_osd(osd_num): - """Starts the specified OSD number. - - :param osd_num: the osd number to start. - """ - if systemd(): - service_start('ceph-osd@{}'.format(osd_num)) - else: - service_start('ceph-osd', id=osd_num) - - -def disable_osd(osd_num): - """Disables the specified OSD number. - - Ensures that the specified osd will not be automatically started at the - next reboot of the system. Due to differences between init systems, - this method cannot make any guarantees that the specified osd cannot be - started manually. - - :param osd_num: the osd id which should be disabled. - :raises CalledProcessError: if an error occurs invoking the systemd cmd - to disable the OSD - :raises IOError, OSError: if the attempt to read/remove the ready file in - an upstart enabled system fails - """ - if systemd(): - # When running under systemd, the individual ceph-osd daemons run as - # templated units and can be directly addressed by referring to the - # templated service name ceph-osd@. Additionally, systemd - # allows one to disable a specific templated unit by running the - # 'systemctl disable ceph-osd@' command. When disabled, the - # OSD should remain disabled until re-enabled via systemd. - # Note: disabling an already disabled service in systemd returns 0, so - # no need to check whether it is enabled or not. - cmd = ['systemctl', 'disable', 'ceph-osd@{}'.format(osd_num)] - subprocess.check_call(cmd) - else: - # Neither upstart nor the ceph-osd upstart script provides for - # disabling the starting of an OSD automatically. The specific OSD - # cannot be prevented from running manually, however it can be - # prevented from running automatically on reboot by removing the - # 'ready' file in the OSD's root directory. This is due to the - # ceph-osd-all upstart script checking for the presence of this file - # before starting the OSD. - ready_file = os.path.join(OSD_BASE_DIR, 'ceph-{}'.format(osd_num), - 'ready') - if os.path.exists(ready_file): - os.unlink(ready_file) - - -def enable_osd(osd_num): - """Enables the specified OSD number. - - Ensures that the specified osd_num will be enabled and ready to start - automatically in the event of a reboot. - - :param osd_num: the osd id which should be enabled. - :raises CalledProcessError: if the call to the systemd command issued - fails when enabling the service - :raises IOError: if the attempt to write the ready file in an usptart - enabled system fails - """ - if systemd(): - cmd = ['systemctl', 'enable', 'ceph-osd@{}'.format(osd_num)] - subprocess.check_call(cmd) - else: - # When running on upstart, the OSDs are started via the ceph-osd-all - # upstart script which will only start the osd if it has a 'ready' - # file. Make sure that file exists. - ready_file = os.path.join(OSD_BASE_DIR, 'ceph-{}'.format(osd_num), - 'ready') - with open(ready_file, 'w') as f: - f.write('ready') - - # Make sure the correct user owns the file. It shouldn't be necessary - # as the upstart script should run with root privileges, but its better - # to have all the files matching ownership. - update_owner(ready_file) - - -def update_owner(path, recurse_dirs=True): - """Changes the ownership of the specified path. - - Changes the ownership of the specified path to the new ceph daemon user - using the system's native chown functionality. This may take awhile, - so this method will issue a set_status for any changes of ownership which - recurses into directory structures. - - :param path: the path to recursively change ownership for - :param recurse_dirs: boolean indicating whether to recursively change the - ownership of all the files in a path's subtree or to - simply change the ownership of the path. - :raises CalledProcessError: if an error occurs issuing the chown system - command - """ - user = ceph_user() - user_group = '{ceph_user}:{ceph_user}'.format(ceph_user=user) - cmd = ['chown', user_group, path] - if os.path.isdir(path) and recurse_dirs: - status_set('maintenance', ('Updating ownership of %s to %s' % - (path, user))) - cmd.insert(1, '-R') - - log('Changing ownership of {path} to {user}'.format( - path=path, user=user_group), DEBUG) - start = datetime.now() - subprocess.check_call(cmd) - elapsed_time = (datetime.now() - start) - - log('Took {secs} seconds to change the ownership of path: {path}'.format( - secs=elapsed_time.total_seconds(), path=path), DEBUG) - - -def list_pools(service): - """ - This will list the current pools that Ceph has - - :param service: String service id to run under - :return: list. Returns a list of the ceph pools. Raises CalledProcessError - if the subprocess fails to run. - """ - try: - pool_list = [] - pools = check_output(['rados', '--id', service, 'lspools']) - for pool in pools.splitlines(): - pool_list.append(pool) - return pool_list - except subprocess.CalledProcessError as err: - log("rados lspools failed with error: {}".format(err.output)) - raise - - -def dirs_need_ownership_update(service): - """Determines if directories still need change of ownership. - - Examines the set of directories under the /var/lib/ceph/{service} directory - and determines if they have the correct ownership or not. This is - necessary due to the upgrade from Hammer to Jewel where the daemon user - changes from root: to ceph:. - - :param service: the name of the service folder to check (e.g. osd, mon) - :return: boolean. True if the directories need a change of ownership, - False otherwise. - :raises IOError: if an error occurs reading the file stats from one of - the child directories. - :raises OSError: if the specified path does not exist or some other error - """ - expected_owner = expected_group = ceph_user() - path = os.path.join(CEPH_BASE_DIR, service) - for child in _get_child_dirs(path): - curr_owner, curr_group = owner(child) - - if (curr_owner == expected_owner) and (curr_group == expected_group): - continue - - log('Directory "%s" needs its ownership updated' % child, DEBUG) - return True - - # All child directories had the expected ownership - return False - -# A dict of valid ceph upgrade paths. Mapping is old -> new -UPGRADE_PATHS = { - 'firefly': 'hammer', - 'hammer': 'jewel', -} - -# Map UCA codenames to ceph codenames -UCA_CODENAME_MAP = { - 'icehouse': 'firefly', - 'juno': 'firefly', - 'kilo': 'hammer', - 'liberty': 'hammer', - 'mitaka': 'jewel', -} - - -def pretty_print_upgrade_paths(): - '''Pretty print supported upgrade paths for ceph''' - lines = [] - for key, value in UPGRADE_PATHS.iteritems(): - lines.append("{} -> {}".format(key, value)) - return lines - - -def resolve_ceph_version(source): - ''' - Resolves a version of ceph based on source configuration - based on Ubuntu Cloud Archive pockets. - - @param: source: source configuration option of charm - @returns: ceph release codename or None if not resolvable - ''' - os_release = get_os_codename_install_source(source) - return UCA_CODENAME_MAP.get(os_release) - - -def get_ceph_pg_stat(): - """ - Returns the result of ceph pg stat - :return: dict - """ - try: - tree = check_output(['ceph', 'pg', 'stat', '--format=json']) - try: - json_tree = json.loads(tree) - if not json_tree['num_pg_by_state']: - return None - return json_tree - except ValueError as v: - log("Unable to parse ceph pg stat json: {}. Error: {}".format( - tree, v.message)) - raise - except subprocess.CalledProcessError as e: - log("ceph pg stat command failed with message: {}".format( - e.message)) - raise - - -def get_ceph_health(): - """ - Returns the health of the cluster from a 'ceph status' - :return: dict - Also raises CalledProcessError if our ceph command fails - To get the overall status, use get_ceph_health()['overall_status'] - """ - try: - tree = check_output( - ['ceph', 'status', '--format=json']) - try: - json_tree = json.loads(tree) - # Make sure children are present in the json - if not json_tree['overall_status']: - return None - return json_tree - except ValueError as v: - log("Unable to parse ceph tree json: {}. Error: {}".format( - tree, v.message)) - raise - except subprocess.CalledProcessError as e: - log("ceph status command failed with message: {}".format( - e.message)) - raise - - -def reweight_osd(osd_num, new_weight): - """ - Changes the crush weight of an OSD to the value specified. - :param osd_num: the osd id which should be changed - :param new_weight: the new weight for the OSD - :returns: bool. True if output looks right, else false. - :raises CalledProcessError: if an error occurs invoking the systemd cmd - """ - try: - cmd_result = subprocess.check_output( - ['ceph', 'osd', 'crush', 'reweight', "osd.{}".format(osd_num), - new_weight], stderr=subprocess.STDOUT) - expected_result = "reweighted item id {ID} name \'osd.{ID}\'".format( - ID=osd_num) + " to {}".format(new_weight) - log(cmd_result) - if expected_result in cmd_result: - return True - return False - except subprocess.CalledProcessError as e: - log("ceph osd crush reweight command failed with message: {}".format( - e.message)) - raise - - -def determine_packages(): - ''' - Determines packages for installation. - - @returns: list of ceph packages - ''' - if is_container(): - PACKAGES.remove('ntp') - return PACKAGES diff --git a/lib/ceph/ceph_broker.py b/lib/ceph/broker.py similarity index 88% rename from lib/ceph/ceph_broker.py rename to lib/ceph/broker.py index 1f6db8c..b071b91 100644 --- a/lib/ceph/ceph_broker.py +++ b/lib/ceph/broker.py @@ -1,5 +1,3 @@ -#!/usr/bin/python -# # Copyright 2016 Canonical Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,19 +14,21 @@ import json import os + from tempfile import NamedTemporaryFile +from ceph.utils import ( + get_cephfs, + get_osd_weight +) +from ceph.crush_utils import Crushmap + from charmhelpers.core.hookenv import ( log, DEBUG, INFO, ERROR, ) -from ceph import ( - get_cephfs, - get_osd_weight -) -from ceph.ceph_helpers import Crushmap from charmhelpers.contrib.storage.linux.ceph import ( create_erasure_profile, delete_pool, @@ -112,6 +112,9 @@ def process_requests(reqs): This is a versioned api. API version must be supplied by the client making the request. + + :param reqs: dict of request parameters. + :returns: dict. exit-code and reason if not 0 """ request_id = reqs.get('request-id') try: @@ -140,6 +143,12 @@ def process_requests(reqs): def handle_create_erasure_profile(request, service): + """Create an erasure profile. + + :param request: dict of request operations and params + :param service: The ceph client to run the command under. + :returns: dict. exit-code and reason if not 0 + """ # "local" | "shec" or it defaults to "jerasure" erasure_type = request.get('erasure-type') # "host" | "rack" or it defaults to "host" # Any valid Ceph bucket @@ -160,10 +169,9 @@ def handle_create_erasure_profile(request, service): def handle_add_permissions_to_key(request, service): - """ - Groups are defined by the key cephx.groups.(namespace-)?-(name). This key - will contain a dict serialized to JSON with data about the group, including - pools and members. + """Groups are defined by the key cephx.groups.(namespace-)?-(name). This + key will contain a dict serialized to JSON with data about the group, + including pools and members. A group can optionally have a namespace defined that will be used to further restrict pool access. @@ -238,8 +246,7 @@ def pool_permission_list_for_service(service): def get_service_groups(service, namespace=None): - """ - Services are objects stored with some metadata, they look like (for a + """Services are objects stored with some metadata, they look like (for a service named "nova"): { group_names: {'rwx': ['images']}, @@ -272,7 +279,7 @@ def get_service_groups(service, namespace=None): def _build_service_groups(service, namespace=None): - '''Rebuild the 'groups' dict for a service group + """Rebuild the 'groups' dict for a service group :returns: dict: dictionary keyed by group name of the following format: @@ -287,7 +294,7 @@ def _build_service_groups(service, namespace=None): services: ['nova'] } } - ''' + """ all_groups = {} for _, groups in service['group_names'].items(): for group in groups: @@ -299,8 +306,7 @@ def _build_service_groups(service, namespace=None): def get_group(group_name): - """ - A group is a structure to hold data about a named group, structured as: + """A group is a structure to hold data about a named group, structured as: { pools: ['glance'], services: ['nova'] @@ -344,6 +350,12 @@ def get_group_key(group_name): def handle_erasure_pool(request, service): + """Create a new erasure coded pool. + + :param request: dict of request operations and params. + :param service: The ceph client to run the command under. + :returns: dict. exit-code and reason if not 0. + """ pool_name = request.get('name') erasure_profile = request.get('erasure-profile') quota = request.get('max-bytes') @@ -390,6 +402,12 @@ def handle_erasure_pool(request, service): def handle_replicated_pool(request, service): + """Create a new replicated pool. + + :param request: dict of request operations and params. + :param service: The ceph client to run the command under. + :returns: dict. exit-code and reason if not 0. + """ pool_name = request.get('name') replicas = request.get('replicas') quota = request.get('max-bytes') @@ -441,6 +459,13 @@ def handle_replicated_pool(request, service): def handle_create_cache_tier(request, service): + """Create a cache tier on a cold pool. Modes supported are + "writeback" and "readonly". + + :param request: dict of request operations and params + :param service: The ceph client to run the command under. + :returns: dict. exit-code and reason if not 0 + """ # mode = "writeback" | "readonly" storage_pool = request.get('cold-pool') cache_pool = request.get('hot-pool') @@ -462,6 +487,12 @@ def handle_create_cache_tier(request, service): def handle_remove_cache_tier(request, service): + """Remove a cache tier from the cold pool. + + :param request: dict of request operations and params + :param service: The ceph client to run the command under. + :returns: dict. exit-code and reason if not 0 + """ storage_pool = request.get('cold-pool') cache_pool = request.get('hot-pool') # cache and storage pool must exist first @@ -477,6 +508,12 @@ def handle_remove_cache_tier(request, service): def handle_set_pool_value(request, service): + """Sets an arbitrary pool value. + + :param request: dict of request operations and params + :param service: The ceph client to run the command under. + :returns: dict. exit-code and reason if not 0 + """ # Set arbitrary pool values params = {'pool': request.get('name'), 'key': request.get('key'), @@ -501,6 +538,12 @@ def handle_set_pool_value(request, service): def handle_rgw_regionmap_update(request, service): + """Change the radosgw region map. + + :param request: dict of request operations and params + :param service: The ceph client to run the command under. + :returns: dict. exit-code and reason if not 0 + """ name = request.get('client-name') if not name: msg = "Missing rgw-region or client-name params" @@ -516,6 +559,12 @@ def handle_rgw_regionmap_update(request, service): def handle_rgw_regionmap_default(request, service): + """Create a radosgw region map. + + :param request: dict of request operations and params + :param service: The ceph client to run the command under. + :returns: dict. exit-code and reason if not 0 + """ region = request.get('rgw-region') name = request.get('client-name') if not region or not name: @@ -537,6 +586,12 @@ def handle_rgw_regionmap_default(request, service): def handle_rgw_zone_set(request, service): + """Create a radosgw zone. + + :param request: dict of request operations and params + :param service: The ceph client to run the command under. + :returns: dict. exit-code and reason if not 0 + """ json_file = request.get('zone-json') name = request.get('client-name') region_name = request.get('region-name') @@ -567,6 +622,12 @@ def handle_rgw_zone_set(request, service): def handle_put_osd_in_bucket(request, service): + """Move an osd into a specified crush bucket. + + :param request: dict of request operations and params + :param service: The ceph client to run the command under. + :returns: dict. exit-code and reason if not 0 + """ osd_id = request.get('osd') target_bucket = request.get('bucket') if not osd_id or not target_bucket: @@ -597,6 +658,12 @@ def handle_put_osd_in_bucket(request, service): def handle_rgw_create_user(request, service): + """Create a new rados gateway user. + + :param request: dict of request operations and params + :param service: The ceph client to run the command under. + :returns: dict. exit-code and reason if not 0 + """ user_id = request.get('rgw-uid') display_name = request.get('display-name') name = request.get('client-name') @@ -630,11 +697,11 @@ def handle_rgw_create_user(request, service): def handle_create_cephfs(request, service): - """ - Create a new cephfs. + """Create a new cephfs. + :param request: The broker request - :param service: The cephx user to run this command under - :return: + :param service: The ceph client to run the command under. + :returns: dict. exit-code and reason if not 0 """ cephfs_name = request.get('mds_name') data_pool = request.get('data_pool') @@ -678,6 +745,12 @@ def handle_create_cephfs(request, service): def handle_rgw_region_set(request, service): # radosgw-admin region set --infile us.json --name client.radosgw.us-east-1 + """Set the rados gateway region. + + :param request: dict. The broker request. + :param service: The ceph client to run the command under. + :returns: dict. exit-code and reason if not 0 + """ json_file = request.get('region-json') name = request.get('client-name') region_name = request.get('region-name') diff --git a/lib/ceph/ceph_helpers.py b/lib/ceph/ceph_helpers.py deleted file mode 100644 index 11f5dd8..0000000 --- a/lib/ceph/ceph_helpers.py +++ /dev/null @@ -1,1557 +0,0 @@ -# Copyright 2014-2015 Canonical Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# -# Copyright 2012 Canonical Ltd. -# -# This file is sourced from lp:openstack-charm-helpers -# -# Authors: -# James Page -# Adam Gandelman -# - -import errno -import hashlib -import math -from charmhelpers.contrib.network.ip import format_ipv6_addr -import six - -import os -import shutil -import json -import time -import uuid -import re - -import subprocess -from subprocess import ( - check_call, - check_output as s_check_output, - CalledProcessError, -) -from charmhelpers.core.hookenv import (config, - local_unit, - relation_get, - relation_ids, - relation_set, - related_units, - log, - DEBUG, - INFO, - WARNING, - ERROR, ) -from charmhelpers.core.host import (mount, - mounts, - service_start, - service_stop, - service_running, - umount, ) -from charmhelpers.fetch import (apt_install, ) - -from charmhelpers.core.kernel import modprobe -from charmhelpers.contrib.openstack.utils import config_flags_parser, \ - get_host_ip - -KEYRING = '/etc/ceph/ceph.client.{}.keyring' -KEYFILE = '/etc/ceph/ceph.client.{}.key' - -CEPH_CONF = """[global] -auth supported = {auth} -keyring = {keyring} -mon host = {mon_hosts} -log to syslog = {use_syslog} -err to syslog = {use_syslog} -clog to syslog = {use_syslog} -""" - -CRUSH_BUCKET = """root {name} {{ - id {id} # do not change unnecessarily - # weight 0.000 - alg straw - hash 0 # rjenkins1 -}} - -rule {name} {{ - ruleset 0 - type replicated - min_size 1 - max_size 10 - step take {name} - step chooseleaf firstn 0 type host - step emit -}}""" - -# This regular expression looks for a string like: -# root NAME { -# id NUMBER -# so that we can extract NAME and ID from the crushmap -CRUSHMAP_BUCKETS_RE = re.compile(r"root\s+(.+)\s+\{\s*id\s+(-?\d+)") - -# This regular expression looks for ID strings in the crushmap like: -# id NUMBER -# so that we can extract the IDs from a crushmap -CRUSHMAP_ID_RE = re.compile(r"id\s+(-?\d+)") - -# The number of placement groups per OSD to target for placement group -# calculations. This number is chosen as 100 due to the ceph PG Calc -# documentation recommending to choose 100 for clusters which are not -# expected to increase in the foreseeable future. Since the majority of the -# calculations are done on deployment, target the case of non-expanding -# clusters as the default. -DEFAULT_PGS_PER_OSD_TARGET = 100 -DEFAULT_POOL_WEIGHT = 10.0 -LEGACY_PG_COUNT = 200 - - -def check_output(*args, **kwargs): - ''' - Helper wrapper for py2/3 compat with subprocess.check_output - - @returns str: UTF-8 decoded representation of output - ''' - return s_check_output(*args, **kwargs).decode('UTF-8') - - -def validator(value, valid_type, valid_range=None): - """ - Used to validate these: http://docs.ceph.com/docs/master/rados/operations/ - pools/#set-pool-values - Example input: - validator(value=1, - valid_type=int, - valid_range=[0, 2]) - This says I'm testing value=1. It must be an int inclusive in [0,2] - - :param value: The value to validate - :param valid_type: The type that value should be. - :param valid_range: A range of values that value can assume. - :return: - """ - assert isinstance(value, valid_type), "{} is not a {}".format(value, - valid_type) - if valid_range is not None: - assert isinstance(valid_range, list), \ - "valid_range must be a list, was given {}".format(valid_range) - # If we're dealing with strings - if valid_type is six.string_types: - assert value in valid_range, \ - "{} is not in the list {}".format(value, valid_range) - # Integer, float should have a min and max - else: - if len(valid_range) != 2: - raise ValueError("Invalid valid_range list of {} for {}. " - "List must be [min,max]".format(valid_range, - value)) - assert value >= valid_range[0], \ - "{} is less than minimum allowed value of {}".format( - value, valid_range[0]) - assert value <= valid_range[1], \ - "{} is greater than maximum allowed value of {}".format( - value, valid_range[1]) - - -class PoolCreationError(Exception): - """ - A custom error to inform the caller that a pool creation failed. Provides - an error message - """ - - def __init__(self, message): - super(PoolCreationError, self).__init__(message) - - -class Crushmap(object): - """An object oriented approach to Ceph crushmap management.""" - - def __init__(self): - """Iiitialize the Crushmap from Ceph""" - self._crushmap = self.load_crushmap() - roots = re.findall(CRUSHMAP_BUCKETS_RE, self._crushmap) - buckets = [] - ids = list(map( - lambda x: int(x), - re.findall(CRUSHMAP_ID_RE, self._crushmap))) - ids.sort() - if roots != []: - for root in roots: - buckets.append(Crushmap.Bucket(root[0], root[1], True)) - - self._buckets = buckets - if ids != []: - self._ids = ids - else: - self._ids = [0] - - def load_crushmap(self): - try: - crush = subprocess.Popen( - ('ceph', 'osd', 'getcrushmap'), - stdout=subprocess.PIPE) - return subprocess.check_output( - ('crushtool', '-d', '-'), - stdin=crush.stdout) - except Exception as e: - log("load_crushmap error: {}".format(e)) - raise "Failed to read Crushmap" - - def ensure_bucket_is_present(self, bucket_name): - if bucket_name not in [bucket.name for bucket in self.buckets()]: - self.add_bucket(bucket_name) - self.save() - - def buckets(self): - """Return a list of buckets that are in the Crushmap.""" - return self._buckets - - def add_bucket(self, bucket_name): - """Add a named bucket to Ceph""" - new_id = min(self._ids) - 1 - self._ids.append(new_id) - self._buckets.append(Crushmap.Bucket(bucket_name, new_id)) - - def save(self): - """Persist Crushmap to Ceph""" - try: - crushmap = self.build_crushmap() - compiled = subprocess.Popen( - ('crushtool', '-c', '/dev/stdin', '-o', '/dev/stdout'), - stdin=subprocess.PIPE, - stdout=subprocess.PIPE) - output = compiled.communicate(crushmap)[0] - ceph = subprocess.Popen( - ('ceph', 'osd', 'setcrushmap', '-i', '/dev/stdin'), - stdin=subprocess.PIPE) - ceph_output = ceph.communicate(input=output) - return ceph_output - except Exception as e: - log("save error: {}".format(e)) - raise "Failed to save crushmap" - - def build_crushmap(self): - """Modifies the curent crushmap to include the new buckets""" - tmp_crushmap = self._crushmap - for bucket in self._buckets: - if not bucket.default: - tmp_crushmap = "{}\n\n{}".format( - tmp_crushmap, - Crushmap.bucket_string(bucket.name, bucket.id)) - return tmp_crushmap - - @staticmethod - def bucket_string(name, id): - return CRUSH_BUCKET.format(name=name, id=id) - - class Bucket(object): - """An object that describes a Crush bucket.""" - - def __init__(self, name, id, default=False): - self.name = name - self.id = int(id) - self.default = default - - def __repr__(self): - return "Bucket {{Name: {name}, ID: {id}}}".format( - name=self.name, id=self.id) - - def __eq__(self, other): - """Override the default Equals behavior""" - if isinstance(other, self.__class__): - return self.__dict__ == other.__dict__ - return NotImplemented - - def __ne__(self, other): - """Define a non-equality test""" - if isinstance(other, self.__class__): - return not self.__eq__(other) - return NotImplemented - - -class Pool(object): - """ - An object oriented approach to Ceph pool creation. This base class is - inherited by ReplicatedPool and ErasurePool. - Do not call create() on this base class as it will not do anything. - Instantiate a child class and call create(). - """ - - def __init__(self, service, name): - self.service = service - self.name = name - - # Create the pool if it doesn't exist already - # To be implemented by subclasses - def create(self): - pass - - def add_cache_tier(self, cache_pool, mode): - """ - Adds a new cache tier to an existing pool. - :param cache_pool: six.string_types. The cache tier pool name to add. - :param mode: six.string_types. The caching mode to use for this pool. - valid range = ["readonly", "writeback"] - :return: None - """ - # Check the input types and values - validator(value=cache_pool, valid_type=six.string_types) - validator(value=mode, - valid_type=six.string_types, - valid_range=["readonly", "writeback"]) - - check_call(['ceph', '--id', self.service, 'osd', 'tier', 'add', - self.name, cache_pool]) - check_call(['ceph', '--id', self.service, 'osd', 'tier', 'cache-mode', - cache_pool, mode]) - check_call(['ceph', '--id', self.service, 'osd', 'tier', 'set-overlay', - self.name, cache_pool]) - check_call(['ceph', '--id', self.service, 'osd', 'pool', 'set', - cache_pool, 'hit_set_type', 'bloom']) - - def remove_cache_tier(self, cache_pool): - """ - Removes a cache tier from Ceph. Flushes all dirty objects from - writeback pools and waits for that to complete. - :param cache_pool: six.string_types. The cache tier pool name to - remove. - :return: None - """ - # read-only is easy, writeback is much harder - mode = get_cache_mode(self.service, cache_pool) - version = ceph_version() - if mode == 'readonly': - check_call(['ceph', '--id', self.service, 'osd', 'tier', - 'cache-mode', cache_pool, 'none']) - check_call(['ceph', '--id', self.service, 'osd', 'tier', 'remove', - self.name, cache_pool]) - - elif mode == 'writeback': - pool_forward_cmd = ['ceph', '--id', self.service, 'osd', 'tier', - 'cache-mode', cache_pool, 'forward'] - if version >= '10.1': - # Jewel added a mandatory flag - pool_forward_cmd.append('--yes-i-really-mean-it') - - check_call(pool_forward_cmd) - # Flush the cache and wait for it to return - check_call(['rados', '--id', self.service, '-p', cache_pool, - 'cache-flush-evict-all']) - check_call(['ceph', '--id', self.service, 'osd', 'tier', - 'remove-overlay', self.name]) - check_call(['ceph', '--id', self.service, 'osd', 'tier', 'remove', - self.name, cache_pool]) - - def get_pgs(self, pool_size, percent_data=DEFAULT_POOL_WEIGHT): - """Return the number of placement groups to use when creating the pool. - - Returns the number of placement groups which should be specified when - creating the pool. This is based upon the calculation guidelines - provided by the Ceph Placement Group Calculator (located online at - http://ceph.com/pgcalc/). - - The number of placement groups are calculated using the following: - - (Target PGs per OSD) * (OSD #) * (%Data) - ---------------------------------------- - (Pool size) - - Per the upstream guidelines, the OSD # should really be considered - based on the number of OSDs which are eligible to be selected by the - pool. Since the pool creation doesn't specify any of CRUSH set rules, - the default rule will be dependent upon the type of pool being - created (replicated or erasure). - - This code makes no attempt to determine the number of OSDs which can be - selected for the specific rule, rather it is left to the user to tune - in the form of 'expected-osd-count' config option. - - :param pool_size: int. pool_size is either the number of replicas for - replicated pools or the K+M sum for erasure coded pools - :param percent_data: float. the percentage of data that is expected to - be contained in the pool for the specific OSD set. Default value - is to assume 10% of the data is for this pool, which is a - relatively low % of the data but allows for the pg_num to be - increased. NOTE: the default is primarily to handle the scenario - where related charms requiring pools has not been upgraded to - include an update to indicate their relative usage of the pools. - :return: int. The number of pgs to use. - """ - - # Note: This calculation follows the approach that is provided - # by the Ceph PG Calculator located at http://ceph.com/pgcalc/. - validator(value=pool_size, valid_type=int) - - # Ensure that percent data is set to something - even with a default - # it can be set to None, which would wreak havoc below. - if percent_data is None: - percent_data = DEFAULT_POOL_WEIGHT - - # If the expected-osd-count is specified, then use the max between - # the expected-osd-count and the actual osd_count - osd_list = get_osds(self.service) - expected = config('expected-osd-count') or 0 - - if osd_list: - osd_count = max(expected, len(osd_list)) - - # Log a message to provide some insight if the calculations claim - # to be off because someone is setting the expected count and - # there are more OSDs in reality. Try to make a proper guess - # based upon the cluster itself. - if expected and osd_count != expected: - log("Found more OSDs than provided expected count. " - "Using the actual count instead", INFO) - elif expected: - # Use the expected-osd-count in older ceph versions to allow for - # a more accurate pg calculations - osd_count = expected - else: - # NOTE(james-page): Default to 200 for older ceph versions - # which don't support OSD query from cli - return LEGACY_PG_COUNT - - percent_data /= 100.0 - target_pgs_per_osd = config( - 'pgs-per-osd') or DEFAULT_PGS_PER_OSD_TARGET - num_pg = (target_pgs_per_osd * osd_count * percent_data) // pool_size - - # The CRUSH algorithm has a slight optimization for placement groups - # with powers of 2 so find the nearest power of 2. If the nearest - # power of 2 is more than 25% below the original value, the next - # highest value is used. To do this, find the nearest power of 2 such - # that 2^n <= num_pg, check to see if its within the 25% tolerance. - exponent = math.floor(math.log(num_pg, 2)) - nearest = 2 ** exponent - if (num_pg - nearest) > (num_pg * 0.25): - # Choose the next highest power of 2 since the nearest is more - # than 25% below the original value. - return int(nearest * 2) - else: - return int(nearest) - - -class ReplicatedPool(Pool): - def __init__(self, - service, - name, - pg_num=None, - replicas=2, - percent_data=10.0): - super(ReplicatedPool, self).__init__(service=service, name=name) - self.replicas = replicas - if pg_num: - # Since the number of placement groups were specified, ensure - # that there aren't too many created. - max_pgs = self.get_pgs(self.replicas, 100.0) - self.pg_num = min(pg_num, max_pgs) - else: - self.pg_num = self.get_pgs(self.replicas, percent_data) - - def create(self): - if not pool_exists(self.service, self.name): - # Create it - cmd = ['ceph', '--id', self.service, 'osd', 'pool', 'create', - self.name, str(self.pg_num)] - try: - check_call(cmd) - # Set the pool replica size - update_pool(client=self.service, - pool=self.name, - settings={'size': str(self.replicas)}) - except CalledProcessError: - raise - - -# Default jerasure erasure coded pool -class ErasurePool(Pool): - def __init__(self, - service, - name, - erasure_code_profile="default", - percent_data=10.0): - super(ErasurePool, self).__init__(service=service, name=name) - self.erasure_code_profile = erasure_code_profile - self.percent_data = percent_data - - def create(self): - if not pool_exists(self.service, self.name): - # Try to find the erasure profile information in order to properly - # size the number of placement groups. The size of an erasure - # coded placement group is calculated as k+m. - erasure_profile = get_erasure_profile(self.service, - self.erasure_code_profile) - - # Check for errors - if erasure_profile is None: - msg = ("Failed to discover erasure profile named " - "{}".format(self.erasure_code_profile)) - log(msg, level=ERROR) - raise PoolCreationError(msg) - if 'k' not in erasure_profile or 'm' not in erasure_profile: - # Error - msg = ("Unable to find k (data chunks) or m (coding chunks) " - "in erasure profile {}".format(erasure_profile)) - log(msg, level=ERROR) - raise PoolCreationError(msg) - - k = int(erasure_profile['k']) - m = int(erasure_profile['m']) - pgs = self.get_pgs(k + m, self.percent_data) - # Create it - cmd = ['ceph', '--id', self.service, 'osd', 'pool', 'create', - self.name, str(pgs), str(pgs), 'erasure', - self.erasure_code_profile] - try: - check_call(cmd) - except CalledProcessError: - raise - - """Get an existing erasure code profile if it already exists. - Returns json formatted output""" - - -def get_mon_map(service): - """ - Returns the current monitor map. - :param service: six.string_types. The Ceph user name to run the command - under - :return: json string. :raise: ValueError if the monmap fails to parse. - Also raises CalledProcessError if our ceph command fails - """ - try: - mon_status = check_output(['ceph', '--id', service, 'mon_status', - '--format=json']) - try: - return json.loads(mon_status) - except ValueError as v: - log("Unable to parse mon_status json: {}. Error: {}".format( - mon_status, v.message)) - raise - except CalledProcessError as e: - log("mon_status command failed with message: {}".format(e.message)) - raise - - -def hash_monitor_names(service): - """ - Uses the get_mon_map() function to get information about the monitor - cluster. - Hash the name of each monitor. Return a sorted list of monitor hashes - in an ascending order. - :param service: six.string_types. The Ceph user name to run the command - under - :rtype : dict. json dict of monitor name, ip address and rank - example: { - 'name': 'ip-172-31-13-165', - 'rank': 0, - 'addr': '172.31.13.165:6789/0'} - """ - try: - hash_list = [] - monitor_list = get_mon_map(service=service) - if monitor_list['monmap']['mons']: - for mon in monitor_list['monmap']['mons']: - hash_list.append(hashlib.sha224(mon['name'].encode( - 'utf-8')).hexdigest()) - return sorted(hash_list) - else: - return None - except (ValueError, CalledProcessError): - raise - - -def monitor_key_delete(service, key): - """ - Delete a key and value pair from the monitor cluster - :param service: six.string_types. The Ceph user name to run the command - under - :param key: six.string_types. The key to delete. - """ - try: - check_output(['ceph', '--id', service, - 'config-key', 'del', str(key)]) - except CalledProcessError as e: - log("Monitor config-key put failed with message: {}".format(e.output)) - raise - - -def monitor_key_set(service, key, value): - """ - Sets a key value pair on the monitor cluster. - :param service: six.string_types. The Ceph user name to run the command - under - :param key: six.string_types. The key to set. - :param value: The value to set. This will be converted to a string - before setting - """ - try: - check_output(['ceph', '--id', service, 'config-key', 'put', str(key), - str(value)]) - except CalledProcessError as e: - log("Monitor config-key put failed with message: {}".format(e.output)) - raise - - -def monitor_key_get(service, key): - """ - Gets the value of an existing key in the monitor cluster. - :param service: six.string_types. The Ceph user name to run the command - under - :param key: six.string_types. The key to search for. - :return: Returns the value of that key or None if not found. - """ - try: - output = check_output(['ceph', '--id', service, 'config-key', 'get', - str(key)]) - return output - except CalledProcessError as e: - log("Monitor config-key get failed with message: {}".format(e.output)) - return None - - -def monitor_key_exists(service, key): - """ - Searches for the existence of a key in the monitor cluster. - :param service: six.string_types. The Ceph user name to run the command - under - :param key: six.string_types. The key to search for - :return: Returns True if the key exists, False if not and raises an - exception if an unknown error occurs. :raise: CalledProcessError if - an unknown error occurs - """ - try: - check_call(['ceph', '--id', service, 'config-key', 'exists', str(key)]) - # I can return true here regardless because Ceph returns - # ENOENT if the key wasn't found - return True - except CalledProcessError as e: - if e.returncode == errno.ENOENT: - return False - else: - log("Unknown error from ceph config-get exists: {} {}".format( - e.returncode, e.output)) - raise - - -def get_erasure_profile(service, name): - """ - :param service: six.string_types. The Ceph user name to run the command - under - :param name: - :return: - """ - try: - out = check_output( - ['ceph', '--id', service, 'osd', 'erasure-code-profile', 'get', - name, '--format=json']) - return json.loads(out) - except (CalledProcessError, OSError, ValueError): - return None - - -def pool_set(service, pool_name, key, value): - """ - Sets a value for a RADOS pool in ceph. - :param service: six.string_types. The Ceph user name to run the command - under - :param pool_name: six.string_types - :param key: six.string_types - :param value: - :return: None. Can raise CalledProcessError - """ - cmd = ['ceph', '--id', service, 'osd', 'pool', 'set', pool_name, key, value - ] - try: - check_call(cmd) - except CalledProcessError: - raise - - -def snapshot_pool(service, pool_name, snapshot_name): - """ - Snapshots a RADOS pool in ceph. - :param service: six.string_types. The Ceph user name to run the command - under - :param pool_name: six.string_types - :param snapshot_name: six.string_types - :return: None. Can raise CalledProcessError - """ - cmd = ['ceph', '--id', service, 'osd', 'pool', 'mksnap', pool_name, - snapshot_name] - try: - check_call(cmd) - except CalledProcessError: - raise - - -def remove_pool_snapshot(service, pool_name, snapshot_name): - """ - Remove a snapshot from a RADOS pool in ceph. - :param service: six.string_types. The Ceph user name to run the command - under - :param pool_name: six.string_types - :param snapshot_name: six.string_types - :return: None. Can raise CalledProcessError - """ - cmd = ['ceph', '--id', service, 'osd', 'pool', 'rmsnap', pool_name, - snapshot_name] - try: - check_call(cmd) - except CalledProcessError: - raise - - -# max_bytes should be an int or long -def set_pool_quota(service, pool_name, max_bytes): - """ - :param service: six.string_types. The Ceph user name to run the command - under - :param pool_name: six.string_types - :param max_bytes: int or long - :return: None. Can raise CalledProcessError - """ - # Set a byte quota on a RADOS pool in ceph. - cmd = ['ceph', '--id', service, 'osd', 'pool', 'set-quota', pool_name, - 'max_bytes', str(max_bytes)] - try: - check_call(cmd) - except CalledProcessError: - raise - - -def remove_pool_quota(service, pool_name): - """ - Set a byte quota on a RADOS pool in ceph. - :param service: six.string_types. The Ceph user name to run the command - under - :param pool_name: six.string_types - :return: None. Can raise CalledProcessError - """ - cmd = ['ceph', '--id', service, 'osd', 'pool', 'set-quota', pool_name, - 'max_bytes', '0'] - try: - check_call(cmd) - except CalledProcessError: - raise - - -def remove_erasure_profile(service, profile_name): - """ - Create a new erasure code profile if one does not already exist for it. - Updates - the profile if it exists. Please see http://docs.ceph.com/docs/master/ - rados/operations/erasure-code-profile/ - for more details - :param service: six.string_types. The Ceph user name to run the command - under - :param profile_name: six.string_types - :return: None. Can raise CalledProcessError - """ - cmd = ['ceph', '--id', service, 'osd', 'erasure-code-profile', 'rm', - profile_name] - try: - check_call(cmd) - except CalledProcessError: - raise - - -def create_erasure_profile(service, - profile_name, - erasure_plugin_name='jerasure', - failure_domain='host', - data_chunks=2, - coding_chunks=1, - locality=None, - durability_estimator=None): - """ - Create a new erasure code profile if one does not already exist for it. - Updates - the profile if it exists. Please see http://docs.ceph.com/docs/master/ - rados/operations/erasure-code-profile/ - for more details - :param service: six.string_types. The Ceph user name to run the command - under - :param profile_name: six.string_types - :param erasure_plugin_name: six.string_types - :param failure_domain: six.string_types. One of ['chassis', 'datacenter', - 'host', 'osd', 'pdu', 'pod', 'rack', 'region', 'room', 'root', 'row']) - :param data_chunks: int - :param coding_chunks: int - :param locality: int - :param durability_estimator: int - :return: None. Can raise CalledProcessError - """ - # Ensure this failure_domain is allowed by Ceph - validator(failure_domain, six.string_types, - ['chassis', 'datacenter', 'host', 'osd', 'pdu', 'pod', 'rack', - 'region', 'room', 'root', 'row']) - - cmd = ['ceph', '--id', service, 'osd', 'erasure-code-profile', 'set', - profile_name, 'plugin=' + erasure_plugin_name, - 'k=' + str(data_chunks), 'm=' + str(coding_chunks), - 'ruleset_failure_domain=' + failure_domain] - if locality is not None and durability_estimator is not None: - raise ValueError( - "create_erasure_profile should be called with k, m and one of l " - "or c but not both.") - - # Add plugin specific information - if locality is not None: - # For local erasure codes - cmd.append('l=' + str(locality)) - if durability_estimator is not None: - # For Shec erasure codes - cmd.append('c=' + str(durability_estimator)) - - if erasure_profile_exists(service, profile_name): - cmd.append('--force') - - try: - check_call(cmd) - except CalledProcessError: - raise - - -def rename_pool(service, old_name, new_name): - """ - Rename a Ceph pool from old_name to new_name - :param service: six.string_types. The Ceph user name to run the command - under - :param old_name: six.string_types - :param new_name: six.string_types - :return: None - """ - validator(value=old_name, valid_type=six.string_types) - validator(value=new_name, valid_type=six.string_types) - - cmd = ['ceph', '--id', service, 'osd', 'pool', 'rename', old_name, new_name - ] - check_call(cmd) - - -def erasure_profile_exists(service, name): - """ - Check to see if an Erasure code profile already exists. - :param service: six.string_types. The Ceph user name to run the command - under - :param name: six.string_types - :return: int or None - """ - validator(value=name, valid_type=six.string_types) - try: - check_call(['ceph', '--id', service, 'osd', 'erasure-code-profile', - 'get', name]) - return True - except CalledProcessError: - return False - - -def get_cache_mode(service, pool_name): - """ - Find the current caching mode of the pool_name given. - :param service: six.string_types. The Ceph user name to run the command - under - :param pool_name: six.string_types - :return: int or None - """ - validator(value=service, valid_type=six.string_types) - validator(value=pool_name, valid_type=six.string_types) - out = check_output(['ceph', '--id', service, 'osd', 'dump', '--format=json' - ]) - try: - osd_json = json.loads(out) - for pool in osd_json['pools']: - if pool['pool_name'] == pool_name: - return pool['cache_mode'] - return None - except ValueError: - raise - - -def pool_exists(service, name): - """Check to see if a RADOS pool already exists.""" - try: - out = check_output(['rados', '--id', service, 'lspools']) - except CalledProcessError: - return False - - return name in out.split() - - -def get_osds(service): - """Return a list of all Ceph Object Storage Daemons currently in the - cluster. - """ - version = ceph_version() - if version and version >= '0.56': - return json.loads(check_output(['ceph', '--id', service, 'osd', 'ls', - '--format=json'])) - - return None - - -def install(): - """Basic Ceph client installation.""" - ceph_dir = "/etc/ceph" - if not os.path.exists(ceph_dir): - os.mkdir(ceph_dir) - - apt_install('ceph-common', fatal=True) - - -def rbd_exists(service, pool, rbd_img): - """Check to see if a RADOS block device exists.""" - try: - out = check_output(['rbd', 'list', '--id', service, '--pool', pool - ]) - except CalledProcessError: - return False - - return rbd_img in out - - -def create_rbd_image(service, pool, image, sizemb): - """Create a new RADOS block device.""" - cmd = ['rbd', 'create', image, '--size', str(sizemb), '--id', service, - '--pool', pool] - check_call(cmd) - - -def update_pool(client, pool, settings): - cmd = ['ceph', '--id', client, 'osd', 'pool', 'set', pool] - for k, v in six.iteritems(settings): - cmd.append(k) - cmd.append(v) - - check_call(cmd) - - -def create_pool(service, name, replicas=3, pg_num=None): - """Create a new RADOS pool.""" - if pool_exists(service, name): - log("Ceph pool {} already exists, skipping creation".format(name), - level=WARNING) - return - - if not pg_num: - # Calculate the number of placement groups based - # on upstream recommended best practices. - osds = get_osds(service) - if osds: - pg_num = (len(osds) * 100 // replicas) - else: - # NOTE(james-page): Default to 200 for older ceph versions - # which don't support OSD query from cli - pg_num = 200 - - cmd = ['ceph', '--id', service, 'osd', 'pool', 'create', name, str(pg_num)] - check_call(cmd) - - update_pool(service, name, settings={'size': str(replicas)}) - - -def delete_pool(service, name): - """Delete a RADOS pool from ceph.""" - cmd = ['ceph', '--id', service, 'osd', 'pool', 'delete', name, - '--yes-i-really-really-mean-it'] - check_call(cmd) - - -def _keyfile_path(service): - return KEYFILE.format(service) - - -def _keyring_path(service): - return KEYRING.format(service) - - -def create_keyring(service, key): - """Create a new Ceph keyring containing key.""" - keyring = _keyring_path(service) - if os.path.exists(keyring): - log('Ceph keyring exists at %s.' % keyring, level=WARNING) - return - - cmd = ['ceph-authtool', keyring, '--create-keyring', - '--name=client.{}'.format(service), '--add-key={}'.format(key)] - check_call(cmd) - log('Created new ceph keyring at %s.' % keyring, level=DEBUG) - - -def delete_keyring(service): - """Delete an existing Ceph keyring.""" - keyring = _keyring_path(service) - if not os.path.exists(keyring): - log('Keyring does not exist at %s' % keyring, level=WARNING) - return - - os.remove(keyring) - log('Deleted ring at %s.' % keyring, level=INFO) - - -def create_key_file(service, key): - """Create a file containing key.""" - keyfile = _keyfile_path(service) - if os.path.exists(keyfile): - log('Keyfile exists at %s.' % keyfile, level=WARNING) - return - - with open(keyfile, 'w') as fd: - fd.write(key) - - log('Created new keyfile at %s.' % keyfile, level=INFO) - - -def get_ceph_nodes(relation='ceph'): - """Query named relation to determine current nodes.""" - hosts = [] - for r_id in relation_ids(relation): - for unit in related_units(r_id): - hosts.append(relation_get('private-address', unit=unit, rid=r_id)) - - return hosts - - -def configure(service, key, auth, use_syslog): - """Perform basic configuration of Ceph.""" - create_keyring(service, key) - create_key_file(service, key) - hosts = get_ceph_nodes() - with open('/etc/ceph/ceph.conf', 'w') as ceph_conf: - ceph_conf.write(CEPH_CONF.format(auth=auth, - keyring=_keyring_path(service), - mon_hosts=",".join(map(str, hosts)), - use_syslog=use_syslog)) - modprobe('rbd') - - -def image_mapped(name): - """Determine whether a RADOS block device is mapped locally.""" - try: - out = check_output(['rbd', 'showmapped']) - except CalledProcessError: - return False - - return name in out - - -def map_block_storage(service, pool, image): - """Map a RADOS block device for local use.""" - cmd = [ - 'rbd', - 'map', - '{}/{}'.format(pool, image), - '--user', - service, - '--secret', - _keyfile_path(service), - ] - check_call(cmd) - - -def filesystem_mounted(fs): - """Determine whether a filesytems is already mounted.""" - return fs in [f for f, m in mounts()] - - -def make_filesystem(blk_device, fstype='ext4', timeout=10): - """Make a new filesystem on the specified block device.""" - count = 0 - e_noent = os.errno.ENOENT - while not os.path.exists(blk_device): - if count >= timeout: - log('Gave up waiting on block device %s' % blk_device, level=ERROR) - raise IOError(e_noent, os.strerror(e_noent), blk_device) - - log('Waiting for block device %s to appear' % blk_device, level=DEBUG) - count += 1 - time.sleep(1) - else: - log('Formatting block device %s as filesystem %s.' % - (blk_device, fstype), - level=INFO) - check_call(['mkfs', '-t', fstype, blk_device]) - - -def place_data_on_block_device(blk_device, data_src_dst): - """Migrate data in data_src_dst to blk_device and then remount.""" - # mount block device into /mnt - mount(blk_device, '/mnt') - # copy data to /mnt - copy_files(data_src_dst, '/mnt') - # umount block device - umount('/mnt') - # Grab user/group ID's from original source - _dir = os.stat(data_src_dst) - uid = _dir.st_uid - gid = _dir.st_gid - # re-mount where the data should originally be - # TODO: persist is currently a NO-OP in core.host - mount(blk_device, data_src_dst, persist=True) - # ensure original ownership of new mount. - os.chown(data_src_dst, uid, gid) - - -def copy_files(src, dst, symlinks=False, ignore=None): - """Copy files from src to dst.""" - for item in os.listdir(src): - s = os.path.join(src, item) - d = os.path.join(dst, item) - if os.path.isdir(s): - shutil.copytree(s, d, symlinks, ignore) - else: - shutil.copy2(s, d) - - -def ensure_ceph_storage(service, - pool, - rbd_img, - sizemb, - mount_point, - blk_device, - fstype, - system_services=[], - replicas=3): - """NOTE: This function must only be called from a single service unit for - the same rbd_img otherwise data loss will occur. - - Ensures given pool and RBD image exists, is mapped to a block device, - and the device is formatted and mounted at the given mount_point. - - If formatting a device for the first time, data existing at mount_point - will be migrated to the RBD device before being re-mounted. - - All services listed in system_services will be stopped prior to data - migration and restarted when complete. - """ - # Ensure pool, RBD image, RBD mappings are in place. - if not pool_exists(service, pool): - log('Creating new pool {}.'.format(pool), level=INFO) - create_pool(service, pool, replicas=replicas) - - if not rbd_exists(service, pool, rbd_img): - log('Creating RBD image ({}).'.format(rbd_img), level=INFO) - create_rbd_image(service, pool, rbd_img, sizemb) - - if not image_mapped(rbd_img): - log('Mapping RBD Image {} as a Block Device.'.format(rbd_img), - level=INFO) - map_block_storage(service, pool, rbd_img) - - # make file system - # TODO: What happens if for whatever reason this is run again and - # the data is already in the rbd device and/or is mounted?? - # When it is mounted already, it will fail to make the fs - # XXX: This is really sketchy! Need to at least add an fstab entry - # otherwise this hook will blow away existing data if its executed - # after a reboot. - if not filesystem_mounted(mount_point): - make_filesystem(blk_device, fstype) - - for svc in system_services: - if service_running(svc): - log('Stopping services {} prior to migrating data.' - .format(svc), - level=DEBUG) - service_stop(svc) - - place_data_on_block_device(blk_device, mount_point) - - for svc in system_services: - log('Starting service {} after migrating data.'.format(svc), - level=DEBUG) - service_start(svc) - - -def ensure_ceph_keyring(service, user=None, group=None, relation='ceph'): - """Ensures a ceph keyring is created for a named service and optionally - ensures user and group ownership. - - Returns False if no ceph key is available in relation state. - """ - key = None - for rid in relation_ids(relation): - for unit in related_units(rid): - key = relation_get('key', rid=rid, unit=unit) - if key: - break - - if not key: - return False - - create_keyring(service=service, key=key) - keyring = _keyring_path(service) - if user and group: - check_call(['chown', '%s.%s' % (user, group), keyring]) - - return True - - -def get_mon_hosts(): - """ - Helper function to gather up the ceph monitor host public addresses - :return: list. Returns a list of ip_address:port - """ - hosts = [] - for relid in relation_ids('mon'): - for unit in related_units(relid): - addr = \ - relation_get('ceph-public-address', - unit, - relid) or get_host_ip( - relation_get( - 'private-address', - unit, - relid)) - - if addr: - hosts.append('{}:6789'.format(format_ipv6_addr(addr) or addr)) - - hosts.sort() - return hosts - - -def ceph_version(): - """Retrieve the local version of ceph.""" - if os.path.exists('/usr/bin/ceph'): - cmd = ['ceph', '-v'] - output = check_output(cmd) - output = output.split() - if len(output) > 3: - return output[2] - else: - return None - else: - return None - - -class CephBrokerRq(object): - """Ceph broker request. - - Multiple operations can be added to a request and sent to the Ceph broker - to be executed. - - Request is json-encoded for sending over the wire. - - The API is versioned and defaults to version 1. - """ - - def __init__(self, api_version=1, request_id=None): - self.api_version = api_version - if request_id: - self.request_id = request_id - else: - self.request_id = str(uuid.uuid1()) - self.ops = [] - - def add_op_create_pool( - self, name, replica_count=3, - pg_num=None, weight=None): - """Adds an operation to create a pool. - - @param pg_num setting: optional setting. If not provided, this value - will be calculated by the broker based on how many OSDs are in the - cluster at the time of creation. Note that, if provided, this value - will be capped at the current available maximum. - @param weight: the percentage of data the pool makes up - """ - if pg_num and weight: - raise ValueError('pg_num and weight are mutually exclusive') - - self.ops.append({'op': 'create-pool', - 'name': name, - 'replicas': replica_count, - 'pg_num': pg_num, - 'weight': weight}) - - def set_ops(self, ops): - """Set request ops to provided value. - - Useful for injecting ops that come from a previous request - to allow comparisons to ensure validity. - """ - self.ops = ops - - @property - def request(self): - return json.dumps({'api-version': self.api_version, - 'ops': self.ops, - 'request-id': self.request_id}) - - def _ops_equal(self, other): - if len(self.ops) == len(other.ops): - for req_no in range(0, len(self.ops)): - for key in ['replicas', 'name', 'op', 'pg_num', 'weight']: - if self.ops[req_no].get(key) != other.ops[req_no].get(key): - return False - else: - return False - return True - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - if self.api_version == other.api_version and \ - self._ops_equal(other): - return True - else: - return False - - def __ne__(self, other): - return not self.__eq__(other) - - -class CephBrokerRsp(object): - """Ceph broker response. - - Response is json-decoded and contents provided as methods/properties. - - The API is versioned and defaults to version 1. - """ - - def __init__(self, encoded_rsp): - self.api_version = None - self.rsp = json.loads(encoded_rsp) - - @property - def request_id(self): - return self.rsp.get('request-id') - - @property - def exit_code(self): - return self.rsp.get('exit-code') - - @property - def exit_msg(self): - return self.rsp.get('stderr') - - -# Ceph Broker Conversation: -# If a charm needs an action to be taken by ceph it can create a CephBrokerRq -# and send that request to ceph via the ceph relation. The CephBrokerRq has a -# unique id so that the client can identity which CephBrokerRsp is associated -# with the request. Ceph will also respond to each client unit individually -# creating a response key per client unit eg glance/0 will get a CephBrokerRsp -# via key broker-rsp-glance-0 -# -# To use this the charm can just do something like: -# -# from charmhelpers.contrib.storage.linux.ceph import ( -# send_request_if_needed, -# is_request_complete, -# CephBrokerRq, -# ) -# -# @hooks.hook('ceph-relation-changed') -# def ceph_changed(): -# rq = CephBrokerRq() -# rq.add_op_create_pool(name='poolname', replica_count=3) -# -# if is_request_complete(rq): -# -# else: -# send_request_if_needed(get_ceph_request()) -# -# CephBrokerRq and CephBrokerRsp are serialized into JSON. Below is an example -# of glance having sent a request to ceph which ceph has successfully processed -# 'ceph:8': { -# 'ceph/0': { -# 'auth': 'cephx', -# 'broker-rsp-glance-0': '{"request-id": "0bc7dc54", "exit-code": 0}', -# 'broker_rsp': '{"request-id": "0da543b8", "exit-code": 0}', -# 'ceph-public-address': '10.5.44.103', -# 'key': 'AQCLDttVuHXINhAAvI144CB09dYchhHyTUY9BQ==', -# 'private-address': '10.5.44.103', -# }, -# 'glance/0': { -# 'broker_req': ('{"api-version": 1, "request-id": "0bc7dc54", ' -# '"ops": [{"replicas": 3, "name": "glance", ' -# '"op": "create-pool"}]}'), -# 'private-address': '10.5.44.109', -# }, -# } - - -def get_previous_request(rid): - """Return the last ceph broker request sent on a given relation - - @param rid: Relation id to query for request - """ - request = None - broker_req = relation_get(attribute='broker_req', - rid=rid, - unit=local_unit()) - if broker_req: - request_data = json.loads(broker_req) - request = CephBrokerRq(api_version=request_data['api-version'], - request_id=request_data['request-id']) - request.set_ops(request_data['ops']) - - return request - - -def get_request_states(request, relation='ceph'): - """Return a dict of requests per relation id with their corresponding - completion state. - - This allows a charm, which has a request for ceph, to see whether there is - an equivalent request already being processed and if so what state that - request is in. - - @param request: A CephBrokerRq object - """ - complete = [] - requests = {} - for rid in relation_ids(relation): - complete = False - previous_request = get_previous_request(rid) - if request == previous_request: - sent = True - complete = is_request_complete_for_rid(previous_request, rid) - else: - sent = False - complete = False - - requests[rid] = {'sent': sent, 'complete': complete, } - - return requests - - -def is_request_sent(request, relation='ceph'): - """Check to see if a functionally equivalent request has already been sent - - Returns True if a similair request has been sent - - @param request: A CephBrokerRq object - """ - states = get_request_states(request, relation=relation) - for rid in states.keys(): - if not states[rid]['sent']: - return False - - return True - - -def is_request_complete(request, relation='ceph'): - """Check to see if a functionally equivalent request has already been - completed - - Returns True if a similair request has been completed - - @param request: A CephBrokerRq object - """ - states = get_request_states(request, relation=relation) - for rid in states.keys(): - if not states[rid]['complete']: - return False - - return True - - -def is_request_complete_for_rid(request, rid): - """Check if a given request has been completed on the given relation - - @param request: A CephBrokerRq object - @param rid: Relation ID - """ - broker_key = get_broker_rsp_key() - for unit in related_units(rid): - rdata = relation_get(rid=rid, unit=unit) - if rdata.get(broker_key): - rsp = CephBrokerRsp(rdata.get(broker_key)) - if rsp.request_id == request.request_id: - if not rsp.exit_code: - return True - else: - # The remote unit sent no reply targeted at this unit so either the - # remote ceph cluster does not support unit targeted replies or it - # has not processed our request yet. - if rdata.get('broker_rsp'): - request_data = json.loads(rdata['broker_rsp']) - if request_data.get('request-id'): - log('Ignoring legacy broker_rsp without unit key as remote' - ' service supports unit specific replies', - level=DEBUG) - else: - log('Using legacy broker_rsp as remote service does not ' - 'supports unit specific replies', - level=DEBUG) - rsp = CephBrokerRsp(rdata['broker_rsp']) - if not rsp.exit_code: - return True - - return False - - -def get_broker_rsp_key(): - """Return broker response key for this unit - - This is the key that ceph is going to use to pass request status - information back to this unit - """ - return 'broker-rsp-' + local_unit().replace('/', '-') - - -def send_request_if_needed(request, relation='ceph'): - """Send broker request if an equivalent request has not already been sent - - @param request: A CephBrokerRq object - """ - if is_request_sent(request, relation=relation): - log('Request already sent but not complete, not sending new request', - level=DEBUG) - else: - for rid in relation_ids(relation): - log('Sending request {}'.format(request.request_id), level=DEBUG) - relation_set(relation_id=rid, broker_req=request.request) - - -class CephConfContext(object): - """Ceph config (ceph.conf) context. - - Supports user-provided Ceph configuration settings. Use can provide a - dictionary as the value for the config-flags charm option containing - Ceph configuration settings keyede by their section in ceph.conf. - """ - - def __init__(self, permitted_sections=None): - self.permitted_sections = permitted_sections or [] - - def __call__(self): - conf = config('config-flags') - if not conf: - return {} - - conf = config_flags_parser(conf) - if type(conf) != dict: - log("Provided config-flags is not a dictionary - ignoring", - level=WARNING) - return {} - - permitted = self.permitted_sections - if permitted: - diff = set(conf.keys()).difference(set(permitted)) - if diff: - log("Config-flags contains invalid keys '%s' - they will be " - "ignored" % (', '.join(diff)), - level=WARNING) - - ceph_conf = {} - for key in conf: - if permitted and key not in permitted: - log("Ignoring key '%s'" % key, level=WARNING) - continue - - ceph_conf[key] = conf[key] - - return ceph_conf diff --git a/lib/ceph/crush_utils.py b/lib/ceph/crush_utils.py new file mode 100644 index 0000000..1c777f3 --- /dev/null +++ b/lib/ceph/crush_utils.py @@ -0,0 +1,149 @@ +# Copyright 2014 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +from subprocess import check_output, CalledProcessError + +from charmhelpers.core.hookenv import ( + log, + ERROR, +) + +CRUSH_BUCKET = """root {name} {{ + id {id} # do not change unnecessarily + # weight 0.000 + alg straw + hash 0 # rjenkins1 +}} + +rule {name} {{ + ruleset 0 + type replicated + min_size 1 + max_size 10 + step take {name} + step chooseleaf firstn 0 type host + step emit +}}""" + +# This regular expression looks for a string like: +# root NAME { +# id NUMBER +# so that we can extract NAME and ID from the crushmap +CRUSHMAP_BUCKETS_RE = re.compile(r"root\s+(.+)\s+\{\s*id\s+(-?\d+)") + +# This regular expression looks for ID strings in the crushmap like: +# id NUMBER +# so that we can extract the IDs from a crushmap +CRUSHMAP_ID_RE = re.compile(r"id\s+(-?\d+)") + + +class Crushmap(object): + """An object oriented approach to Ceph crushmap management.""" + + def __init__(self): + self._crushmap = self.load_crushmap() + roots = re.findall(CRUSHMAP_BUCKETS_RE, self._crushmap) + buckets = [] + ids = list(map( + lambda x: int(x), + re.findall(CRUSHMAP_ID_RE, self._crushmap))) + ids.sort() + if roots != []: + for root in roots: + buckets.append(CRUSHBucket(root[0], root[1], True)) + + self._buckets = buckets + if ids != []: + self._ids = ids + else: + self._ids = [0] + + def load_crushmap(self): + try: + crush = check_output(['ceph', 'osd', 'getcrushmap']) + return check_output(['crushtool', '-d', '-'], stdin=crush.stdout) + except CalledProcessError as e: + log("Error occured while loading and decompiling CRUSH map:" + "{}".format(e), ERROR) + raise "Failed to read CRUSH map" + + def ensure_bucket_is_present(self, bucket_name): + if bucket_name not in [bucket.name for bucket in self.buckets()]: + self.add_bucket(bucket_name) + self.save() + + def buckets(self): + """Return a list of buckets that are in the Crushmap.""" + return self._buckets + + def add_bucket(self, bucket_name): + """Add a named bucket to Ceph""" + new_id = min(self._ids) - 1 + self._ids.append(new_id) + self._buckets.append(CRUSHBucket(bucket_name, new_id)) + + def save(self): + """Persist Crushmap to Ceph""" + try: + crushmap = self.build_crushmap() + compiled = check_output(['crushtool', '-c', '/dev/stdin', '-o', + '/dev/stdout'], stdin=crushmap) + ceph_output = check_output(['ceph', 'osd', 'setcrushmap', '-i', + '/dev/stdin'], stdin=compiled) + return ceph_output + except CalledProcessError as e: + log("save error: {}".format(e)) + raise "Failed to save CRUSH map." + + def build_crushmap(self): + """Modifies the current CRUSH map to include the new buckets""" + tmp_crushmap = self._crushmap + for bucket in self._buckets: + if not bucket.default: + tmp_crushmap = "{}\n\n{}".format( + tmp_crushmap, + Crushmap.bucket_string(bucket.name, bucket.id)) + + return tmp_crushmap + + @staticmethod + def bucket_string(name, id): + return CRUSH_BUCKET.format(name=name, id=id) + + +class CRUSHBucket(object): + """CRUSH bucket description object.""" + + def __init__(self, name, id, default=False): + self.name = name + self.id = int(id) + self.default = default + + def __repr__(self): + return "Bucket {{Name: {name}, ID: {id}}}".format( + name=self.name, id=self.id) + + def __eq__(self, other): + """Override the default Equals behavior""" + if isinstance(other, self.__class__): + return self.__dict__ == other.__dict__ + return NotImplemented + + def __ne__(self, other): + """Define a non-equality test""" + if isinstance(other, self.__class__): + return not self.__eq__(other) + return NotImplemented diff --git a/lib/ceph/utils.py b/lib/ceph/utils.py new file mode 100644 index 0000000..b96dabb --- /dev/null +++ b/lib/ceph/utils.py @@ -0,0 +1,2199 @@ +# Copyright 2017 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections +import ctypes +import errno +import json +import os +import pyudev +import random +import re +import socket +import subprocess +import sys +import time +import shutil + +from datetime import datetime + +from charmhelpers.core import hookenv +from charmhelpers.core import templating +from charmhelpers.core.host import ( + chownr, + cmp_pkgrevno, + lsb_release, + mkdir, + mounts, + owner, + service_restart, + service_start, + service_stop, + CompareHostReleases, + is_container, +) +from charmhelpers.core.hookenv import ( + cached, + config, + log, + status_set, + DEBUG, + ERROR, + WARNING, +) +from charmhelpers.fetch import ( + apt_cache, + add_source, apt_install, apt_update +) +from charmhelpers.contrib.storage.linux.ceph import ( + get_mon_map, + monitor_key_set, + monitor_key_exists, + monitor_key_get, +) +from charmhelpers.contrib.storage.linux.utils import ( + is_block_device, + is_device_mounted, + zap_disk, +) +from charmhelpers.contrib.openstack.utils import ( + get_os_codename_install_source, +) + +CEPH_BASE_DIR = os.path.join(os.sep, 'var', 'lib', 'ceph') +OSD_BASE_DIR = os.path.join(CEPH_BASE_DIR, 'osd') +HDPARM_FILE = os.path.join(os.sep, 'etc', 'hdparm.conf') + +LEADER = 'leader' +PEON = 'peon' +QUORUM = [LEADER, PEON] + +PACKAGES = ['ceph', 'gdisk', 'ntp', 'btrfs-tools', 'python-ceph', + 'radosgw', 'xfsprogs', 'python-pyudev'] + +LinkSpeed = { + "BASE_10": 10, + "BASE_100": 100, + "BASE_1000": 1000, + "GBASE_10": 10000, + "GBASE_40": 40000, + "GBASE_100": 100000, + "UNKNOWN": None +} + +# Mapping of adapter speed to sysctl settings +NETWORK_ADAPTER_SYSCTLS = { + # 10Gb + LinkSpeed["GBASE_10"]: { + 'net.core.rmem_default': 524287, + 'net.core.wmem_default': 524287, + 'net.core.rmem_max': 524287, + 'net.core.wmem_max': 524287, + 'net.core.optmem_max': 524287, + 'net.core.netdev_max_backlog': 300000, + 'net.ipv4.tcp_rmem': '10000000 10000000 10000000', + 'net.ipv4.tcp_wmem': '10000000 10000000 10000000', + 'net.ipv4.tcp_mem': '10000000 10000000 10000000' + }, + # Mellanox 10/40Gb + LinkSpeed["GBASE_40"]: { + 'net.ipv4.tcp_timestamps': 0, + 'net.ipv4.tcp_sack': 1, + 'net.core.netdev_max_backlog': 250000, + 'net.core.rmem_max': 4194304, + 'net.core.wmem_max': 4194304, + 'net.core.rmem_default': 4194304, + 'net.core.wmem_default': 4194304, + 'net.core.optmem_max': 4194304, + 'net.ipv4.tcp_rmem': '4096 87380 4194304', + 'net.ipv4.tcp_wmem': '4096 65536 4194304', + 'net.ipv4.tcp_low_latency': 1, + 'net.ipv4.tcp_adv_win_scale': 1 + } +} + + +class Partition(object): + def __init__(self, name, number, size, start, end, sectors, uuid): + """A block device partition. + + :param name: Name of block device + :param number: Partition number + :param size: Capacity of the device + :param start: Starting block + :param end: Ending block + :param sectors: Number of blocks + :param uuid: UUID of the partition + """ + self.name = name, + self.number = number + self.size = size + self.start = start + self.end = end + self.sectors = sectors + self.uuid = uuid + + def __str__(self): + return "number: {} start: {} end: {} sectors: {} size: {} " \ + "name: {} uuid: {}".format(self.number, self.start, + self.end, + self.sectors, self.size, + self.name, self.uuid) + + def __eq__(self, other): + if isinstance(other, self.__class__): + return self.__dict__ == other.__dict__ + return False + + def __ne__(self, other): + return not self.__eq__(other) + + +def unmounted_disks(): + """List of unmounted block devices on the current host.""" + disks = [] + context = pyudev.Context() + for device in context.list_devices(DEVTYPE='disk'): + if device['SUBSYSTEM'] == 'block': + matched = False + for block_type in [u'dm', u'loop', u'ram', u'nbd']: + if block_type in device.device_node: + matched = True + if matched: + continue + disks.append(device.device_node) + log("Found disks: {}".format(disks)) + return [disk for disk in disks if not is_device_mounted(disk)] + + +def save_sysctls(sysctl_dict, save_location): + """Persist the sysctls to the hard drive. + + :param sysctl_dict: dict + :param save_location: path to save the settings to + :raises: IOError if anything goes wrong with writing. + """ + try: + # Persist the settings for reboots + with open(save_location, "w") as fd: + for key, value in sysctl_dict.items(): + fd.write("{}={}\n".format(key, value)) + + except IOError as e: + log("Unable to persist sysctl settings to {}. Error {}".format( + save_location, e.message), level=ERROR) + raise + + +def tune_nic(network_interface): + """This will set optimal sysctls for the particular network adapter. + + :param network_interface: string The network adapter name. + """ + speed = get_link_speed(network_interface) + if speed in NETWORK_ADAPTER_SYSCTLS: + status_set('maintenance', 'Tuning device {}'.format( + network_interface)) + sysctl_file = os.path.join( + os.sep, + 'etc', + 'sysctl.d', + '51-ceph-osd-charm-{}.conf'.format(network_interface)) + try: + log("Saving sysctl_file: {} values: {}".format( + sysctl_file, NETWORK_ADAPTER_SYSCTLS[speed]), + level=DEBUG) + save_sysctls(sysctl_dict=NETWORK_ADAPTER_SYSCTLS[speed], + save_location=sysctl_file) + except IOError as e: + log("Write to /etc/sysctl.d/51-ceph-osd-charm-{} " + "failed. {}".format(network_interface, e.message), + level=ERROR) + + try: + # Apply the settings + log("Applying sysctl settings", level=DEBUG) + subprocess.check_output(["sysctl", "-p", sysctl_file]) + except subprocess.CalledProcessError as err: + log('sysctl -p {} failed with error {}'.format(sysctl_file, + err.output), + level=ERROR) + else: + log("No settings found for network adapter: {}".format( + network_interface), level=DEBUG) + + +def get_link_speed(network_interface): + """This will find the link speed for a given network device. Returns None + if an error occurs. + :param network_interface: string The network adapter interface. + :returns: LinkSpeed + """ + speed_path = os.path.join(os.sep, 'sys', 'class', 'net', + network_interface, 'speed') + # I'm not sure where else we'd check if this doesn't exist + if not os.path.exists(speed_path): + return LinkSpeed["UNKNOWN"] + + try: + with open(speed_path, 'r') as sysfs: + nic_speed = sysfs.readlines() + + # Did we actually read anything? + if not nic_speed: + return LinkSpeed["UNKNOWN"] + + # Try to find a sysctl match for this particular speed + for name, speed in LinkSpeed.items(): + if speed == int(nic_speed[0].strip()): + return speed + # Default to UNKNOWN if we can't find a match + return LinkSpeed["UNKNOWN"] + except IOError as e: + log("Unable to open {path} because of error: {error}".format( + path=speed_path, + error=e.message), level='error') + return LinkSpeed["UNKNOWN"] + + +def persist_settings(settings_dict): + # Write all settings to /etc/hdparm.conf + """ This will persist the hard drive settings to the /etc/hdparm.conf file + + The settings_dict should be in the form of {"uuid": {"key":"value"}} + + :param settings_dict: dict of settings to save + """ + if not settings_dict: + return + + try: + templating.render(source='hdparm.conf', target=HDPARM_FILE, + context=settings_dict) + except IOError as err: + log("Unable to open {path} because of error: {error}".format( + path=HDPARM_FILE, error=err.message), level=ERROR) + except Exception as e: + # The templating.render can raise a jinja2 exception if the + # template is not found. Rather than polluting the import + # space of this charm, simply catch Exception + log('Unable to render {path} due to error: {error}'.format( + path=HDPARM_FILE, error=e.message), level=ERROR) + + +def set_max_sectors_kb(dev_name, max_sectors_size): + """This function sets the max_sectors_kb size of a given block device. + + :param dev_name: Name of the block device to query + :param max_sectors_size: int of the max_sectors_size to save + """ + max_sectors_kb_path = os.path.join('sys', 'block', dev_name, 'queue', + 'max_sectors_kb') + try: + with open(max_sectors_kb_path, 'w') as f: + f.write(max_sectors_size) + except IOError as e: + log('Failed to write max_sectors_kb to {}. Error: {}'.format( + max_sectors_kb_path, e.message), level=ERROR) + + +def get_max_sectors_kb(dev_name): + """This function gets the max_sectors_kb size of a given block device. + + :param dev_name: Name of the block device to query + :returns: int which is either the max_sectors_kb or 0 on error. + """ + max_sectors_kb_path = os.path.join('sys', 'block', dev_name, 'queue', + 'max_sectors_kb') + + # Read in what Linux has set by default + if os.path.exists(max_sectors_kb_path): + try: + with open(max_sectors_kb_path, 'r') as f: + max_sectors_kb = f.read().strip() + return int(max_sectors_kb) + except IOError as e: + log('Failed to read max_sectors_kb to {}. Error: {}'.format( + max_sectors_kb_path, e.message), level=ERROR) + # Bail. + return 0 + return 0 + + +def get_max_hw_sectors_kb(dev_name): + """This function gets the max_hw_sectors_kb for a given block device. + + :param dev_name: Name of the block device to query + :returns: int which is either the max_hw_sectors_kb or 0 on error. + """ + max_hw_sectors_kb_path = os.path.join('sys', 'block', dev_name, 'queue', + 'max_hw_sectors_kb') + # Read in what the hardware supports + if os.path.exists(max_hw_sectors_kb_path): + try: + with open(max_hw_sectors_kb_path, 'r') as f: + max_hw_sectors_kb = f.read().strip() + return int(max_hw_sectors_kb) + except IOError as e: + log('Failed to read max_hw_sectors_kb to {}. Error: {}'.format( + max_hw_sectors_kb_path, e.message), level=ERROR) + return 0 + return 0 + + +def set_hdd_read_ahead(dev_name, read_ahead_sectors=256): + """This function sets the hard drive read ahead. + + :param dev_name: Name of the block device to set read ahead on. + :param read_ahead_sectors: int How many sectors to read ahead. + """ + try: + # Set the read ahead sectors to 256 + log('Setting read ahead to {} for device {}'.format( + read_ahead_sectors, + dev_name)) + subprocess.check_output(['hdparm', + '-a{}'.format(read_ahead_sectors), + dev_name]) + except subprocess.CalledProcessError as e: + log('hdparm failed with error: {}'.format(e.output), + level=ERROR) + + +def get_block_uuid(block_dev): + """This queries blkid to get the uuid for a block device. + + :param block_dev: Name of the block device to query. + :returns: The UUID of the device or None on Error. + """ + try: + block_info = subprocess.check_output( + ['blkid', '-o', 'export', block_dev]) + for tag in block_info.split('\n'): + parts = tag.split('=') + if parts[0] == 'UUID': + return parts[1] + return None + except subprocess.CalledProcessError as err: + log('get_block_uuid failed with error: {}'.format(err.output), + level=ERROR) + return None + + +def check_max_sectors(save_settings_dict, + block_dev, + uuid): + """Tune the max_hw_sectors if needed. + + make sure that /sys/.../max_sectors_kb matches max_hw_sectors_kb or at + least 1MB for spinning disks + If the box has a RAID card with cache this could go much bigger. + + :param save_settings_dict: The dict used to persist settings + :param block_dev: A block device name: Example: /dev/sda + :param uuid: The uuid of the block device + """ + dev_name = None + path_parts = os.path.split(block_dev) + if len(path_parts) == 2: + dev_name = path_parts[1] + else: + log('Unable to determine the block device name from path: {}'.format( + block_dev)) + # Play it safe and bail + return + max_sectors_kb = get_max_sectors_kb(dev_name=dev_name) + max_hw_sectors_kb = get_max_hw_sectors_kb(dev_name=dev_name) + + if max_sectors_kb < max_hw_sectors_kb: + # OK we have a situation where the hardware supports more than Linux is + # currently requesting + config_max_sectors_kb = hookenv.config('max-sectors-kb') + if config_max_sectors_kb < max_hw_sectors_kb: + # Set the max_sectors_kb to the config.yaml value if it is less + # than the max_hw_sectors_kb + log('Setting max_sectors_kb for device {} to {}'.format( + dev_name, config_max_sectors_kb)) + save_settings_dict[ + "drive_settings"][uuid][ + "read_ahead_sect"] = config_max_sectors_kb + set_max_sectors_kb(dev_name=dev_name, + max_sectors_size=config_max_sectors_kb) + else: + # Set to the max_hw_sectors_kb + log('Setting max_sectors_kb for device {} to {}'.format( + dev_name, max_hw_sectors_kb)) + save_settings_dict[ + "drive_settings"][uuid]['read_ahead_sect'] = max_hw_sectors_kb + set_max_sectors_kb(dev_name=dev_name, + max_sectors_size=max_hw_sectors_kb) + else: + log('max_sectors_kb match max_hw_sectors_kb. No change needed for ' + 'device: {}'.format(block_dev)) + + +def tune_dev(block_dev): + """Try to make some intelligent decisions with HDD tuning. Future work will + include optimizing SSDs. + + This function will change the read ahead sectors and the max write + sectors for each block device. + + :param block_dev: A block device name: Example: /dev/sda + """ + uuid = get_block_uuid(block_dev) + if uuid is None: + log('block device {} uuid is None. Unable to save to ' + 'hdparm.conf'.format(block_dev), level=DEBUG) + return + save_settings_dict = {} + log('Tuning device {}'.format(block_dev)) + status_set('maintenance', 'Tuning device {}'.format(block_dev)) + set_hdd_read_ahead(block_dev) + save_settings_dict["drive_settings"] = {} + save_settings_dict["drive_settings"][uuid] = {} + save_settings_dict["drive_settings"][uuid]['read_ahead_sect'] = 256 + + check_max_sectors(block_dev=block_dev, + save_settings_dict=save_settings_dict, + uuid=uuid) + + persist_settings(settings_dict=save_settings_dict) + status_set('maintenance', 'Finished tuning device {}'.format(block_dev)) + + +def ceph_user(): + if get_version() > 1: + return 'ceph' + else: + return "root" + + +class CrushLocation(object): + def __init__(self, + name, + identifier, + host, + rack, + row, + datacenter, + chassis, + root): + self.name = name + self.identifier = identifier + self.host = host + self.rack = rack + self.row = row + self.datacenter = datacenter + self.chassis = chassis + self.root = root + + def __str__(self): + return "name: {} id: {} host: {} rack: {} row: {} datacenter: {} " \ + "chassis :{} root: {}".format(self.name, self.identifier, + self.host, self.rack, self.row, + self.datacenter, self.chassis, + self.root) + + def __eq__(self, other): + return not self.name < other.name and not other.name < self.name + + def __ne__(self, other): + return self.name < other.name or other.name < self.name + + def __gt__(self, other): + return self.name > other.name + + def __ge__(self, other): + return not self.name < other.name + + def __le__(self, other): + return self.name < other.name + + +def get_osd_weight(osd_id): + """Returns the weight of the specified OSD. + + :returns: Float + :raises: ValueError if the monmap fails to parse. + :raises: CalledProcessError if our ceph command fails. + """ + try: + tree = subprocess.check_output( + ['ceph', 'osd', 'tree', '--format=json']) + try: + json_tree = json.loads(tree) + # Make sure children are present in the json + if not json_tree['nodes']: + return None + for device in json_tree['nodes']: + if device['type'] == 'osd' and device['name'] == osd_id: + return device['crush_weight'] + except ValueError as v: + log("Unable to parse ceph tree json: {}. Error: {}".format( + tree, v.message)) + raise + except subprocess.CalledProcessError as e: + log("ceph osd tree command failed with message: {}".format( + e.message)) + raise + + +def get_osd_tree(service): + """Returns the current osd map in JSON. + + :returns: List. + :raises: ValueError if the monmap fails to parse. + Also raises CalledProcessError if our ceph command fails + """ + try: + tree = subprocess.check_output( + ['ceph', '--id', service, + 'osd', 'tree', '--format=json']) + try: + json_tree = json.loads(tree) + crush_list = [] + # Make sure children are present in the json + if not json_tree['nodes']: + return None + child_ids = json_tree['nodes'][0]['children'] + for child in json_tree['nodes']: + if child['id'] in child_ids: + crush_list.append( + CrushLocation( + name=child.get('name'), + identifier=child['id'], + host=child.get('host'), + rack=child.get('rack'), + row=child.get('row'), + datacenter=child.get('datacenter'), + chassis=child.get('chassis'), + root=child.get('root') + ) + ) + return crush_list + except ValueError as v: + log("Unable to parse ceph tree json: {}. Error: {}".format( + tree, v.message)) + raise + except subprocess.CalledProcessError as e: + log("ceph osd tree command failed with message: {}".format( + e.message)) + raise + + +def _get_child_dirs(path): + """Returns a list of directory names in the specified path. + + :param path: a full path listing of the parent directory to return child + directory names + :returns: list. A list of child directories under the parent directory + :raises: ValueError if the specified path does not exist or is not a + directory, + OSError if an error occurs reading the directory listing + """ + if not os.path.exists(path): + raise ValueError('Specfied path "%s" does not exist' % path) + if not os.path.isdir(path): + raise ValueError('Specified path "%s" is not a directory' % path) + + files_in_dir = [os.path.join(path, f) for f in os.listdir(path)] + return list(filter(os.path.isdir, files_in_dir)) + + +def _get_osd_num_from_dirname(dirname): + """Parses the dirname and returns the OSD id. + + Parses a string in the form of 'ceph-{osd#}' and returns the osd number + from the directory name. + + :param dirname: the directory name to return the OSD number from + :return int: the osd number the directory name corresponds to + :raises ValueError: if the osd number cannot be parsed from the provided + directory name. + """ + match = re.search('ceph-(?P\d+)', dirname) + if not match: + raise ValueError("dirname not in correct format: %s" % dirname) + + return match.group('osd_id') + + +def get_local_osd_ids(): + """This will list the /var/lib/ceph/osd/* directories and try + to split the ID off of the directory name and return it in + a list. + + :returns: list. A list of osd identifiers + :raises: OSError if something goes wrong with listing the directory. + """ + osd_ids = [] + osd_path = os.path.join(os.sep, 'var', 'lib', 'ceph', 'osd') + if os.path.exists(osd_path): + try: + dirs = os.listdir(osd_path) + for osd_dir in dirs: + osd_id = osd_dir.split('-')[1] + if _is_int(osd_id): + osd_ids.append(osd_id) + except OSError: + raise + return osd_ids + + +def get_local_mon_ids(): + """This will list the /var/lib/ceph/mon/* directories and try + to split the ID off of the directory name and return it in + a list. + + :returns: list. A list of monitor identifiers + :raises: OSError if something goes wrong with listing the directory. + """ + mon_ids = [] + mon_path = os.path.join(os.sep, 'var', 'lib', 'ceph', 'mon') + if os.path.exists(mon_path): + try: + dirs = os.listdir(mon_path) + for mon_dir in dirs: + # Basically this takes everything after ceph- as the monitor ID + match = re.search('ceph-(?P.*)', mon_dir) + if match: + mon_ids.append(match.group('mon_id')) + except OSError: + raise + return mon_ids + + +def _is_int(v): + """Return True if the object v can be turned into an integer.""" + try: + int(v) + return True + except ValueError: + return False + + +def get_version(): + """Derive Ceph release from an installed package.""" + import apt_pkg as apt + + cache = apt_cache() + package = "ceph" + try: + pkg = cache[package] + except: + # the package is unknown to the current apt cache. + e = 'Could not determine version of package with no installation ' \ + 'candidate: %s' % package + error_out(e) + + if not pkg.current_ver: + # package is known, but no version is currently installed. + e = 'Could not determine version of uninstalled package: %s' % package + error_out(e) + + vers = apt.upstream_version(pkg.current_ver.ver_str) + + # x.y match only for 20XX.X + # and ignore patch level for other packages + match = re.match('^(\d+)\.(\d+)', vers) + + if match: + vers = match.group(0) + return float(vers) + + +def error_out(msg): + log("FATAL ERROR: %s" % msg, + level=ERROR) + sys.exit(1) + + +def is_quorum(): + asok = "/var/run/ceph/ceph-mon.{}.asok".format(socket.gethostname()) + cmd = [ + "sudo", + "-u", + ceph_user(), + "ceph", + "--admin-daemon", + asok, + "mon_status" + ] + if os.path.exists(asok): + try: + result = json.loads(subprocess.check_output(cmd)) + except subprocess.CalledProcessError: + return False + except ValueError: + # Non JSON response from mon_status + return False + if result['state'] in QUORUM: + return True + else: + return False + else: + return False + + +def is_leader(): + asok = "/var/run/ceph/ceph-mon.{}.asok".format(socket.gethostname()) + cmd = [ + "sudo", + "-u", + ceph_user(), + "ceph", + "--admin-daemon", + asok, + "mon_status" + ] + if os.path.exists(asok): + try: + result = json.loads(subprocess.check_output(cmd)) + except subprocess.CalledProcessError: + return False + except ValueError: + # Non JSON response from mon_status + return False + if result['state'] == LEADER: + return True + else: + return False + else: + return False + + +def wait_for_quorum(): + while not is_quorum(): + log("Waiting for quorum to be reached") + time.sleep(3) + + +def add_bootstrap_hint(peer): + asok = "/var/run/ceph/ceph-mon.{}.asok".format(socket.gethostname()) + cmd = [ + "sudo", + "-u", + ceph_user(), + "ceph", + "--admin-daemon", + asok, + "add_bootstrap_peer_hint", + peer + ] + if os.path.exists(asok): + # Ignore any errors for this call + subprocess.call(cmd) + + +DISK_FORMATS = [ + 'xfs', + 'ext4', + 'btrfs' +] + +CEPH_PARTITIONS = [ + '89C57F98-2FE5-4DC0-89C1-5EC00CEFF2BE', # ceph encrypted disk in creation + '45B0969E-9B03-4F30-B4C6-5EC00CEFF106', # ceph encrypted journal + '4FBD7E29-9D25-41B8-AFD0-5EC00CEFF05D', # ceph encrypted osd data + '4FBD7E29-9D25-41B8-AFD0-062C0CEFF05D', # ceph osd data + '45B0969E-9B03-4F30-B4C6-B4B80CEFF106', # ceph osd journal + '89C57F98-2FE5-4DC0-89C1-F3AD0CEFF2BE', # ceph disk in creation +] + + +def umount(mount_point): + """This function unmounts a mounted directory forcibly. This will + be used for unmounting broken hard drive mounts which may hang. + + If umount returns EBUSY this will lazy unmount. + + :param mount_point: str. A String representing the filesystem mount point + :returns: int. Returns 0 on success. errno otherwise. + """ + libc_path = ctypes.util.find_library("c") + libc = ctypes.CDLL(libc_path, use_errno=True) + + # First try to umount with MNT_FORCE + ret = libc.umount(mount_point, 1) + if ret < 0: + err = ctypes.get_errno() + if err == errno.EBUSY: + # Detach from try. IE lazy umount + ret = libc.umount(mount_point, 2) + if ret < 0: + err = ctypes.get_errno() + return err + return 0 + else: + return err + return 0 + + +def replace_osd(dead_osd_number, + dead_osd_device, + new_osd_device, + osd_format, + osd_journal, + reformat_osd=False, + ignore_errors=False): + """This function will automate the replacement of a failed osd disk as much + as possible. It will revoke the keys for the old osd, remove it from the + crush map and then add a new osd into the cluster. + + :param dead_osd_number: The osd number found in ceph osd tree. Example: 99 + :param dead_osd_device: The physical device. Example: /dev/sda + :param osd_format: + :param osd_journal: + :param reformat_osd: + :param ignore_errors: + """ + host_mounts = mounts() + mount_point = None + for mount in host_mounts: + if mount[1] == dead_osd_device: + mount_point = mount[0] + # need to convert dev to osd number + # also need to get the mounted drive so we can tell the admin to + # replace it + try: + # Drop this osd out of the cluster. This will begin a + # rebalance operation + status_set('maintenance', 'Removing osd {}'.format(dead_osd_number)) + subprocess.check_output([ + 'ceph', + '--id', + 'osd-upgrade', + 'osd', 'out', + 'osd.{}'.format(dead_osd_number)]) + + # Kill the osd process if it's not already dead + if systemd(): + service_stop('ceph-osd@{}'.format(dead_osd_number)) + else: + subprocess.check_output(['stop', 'ceph-osd', 'id={}'.format( + dead_osd_number)]) + # umount if still mounted + ret = umount(mount_point) + if ret < 0: + raise RuntimeError('umount {} failed with error: {}'.format( + mount_point, os.strerror(ret))) + # Clean up the old mount point + shutil.rmtree(mount_point) + subprocess.check_output([ + 'ceph', + '--id', + 'osd-upgrade', + 'osd', 'crush', 'remove', + 'osd.{}'.format(dead_osd_number)]) + # Revoke the OSDs access keys + subprocess.check_output([ + 'ceph', + '--id', + 'osd-upgrade', + 'auth', 'del', + 'osd.{}'.format(dead_osd_number)]) + subprocess.check_output([ + 'ceph', + '--id', + 'osd-upgrade', + 'osd', 'rm', + 'osd.{}'.format(dead_osd_number)]) + status_set('maintenance', 'Setting up replacement osd {}'.format( + new_osd_device)) + osdize(new_osd_device, + osd_format, + osd_journal, + reformat_osd, + ignore_errors) + except subprocess.CalledProcessError as e: + log('replace_osd failed with error: ' + e.output) + + +def get_partition_list(dev): + """Lists the partitions of a block device. + + :param dev: Path to a block device. ex: /dev/sda + :returns: Returns a list of Partition objects. + :raises: CalledProcessException if lsblk fails + """ + partitions_list = [] + try: + partitions = get_partitions(dev) + # For each line of output + for partition in partitions: + parts = partition.split() + partitions_list.append( + Partition(number=parts[0], + start=parts[1], + end=parts[2], + sectors=parts[3], + size=parts[4], + name=parts[5], + uuid=parts[6]) + ) + return partitions_list + except subprocess.CalledProcessError: + raise + + +def is_osd_disk(dev): + partitions = get_partition_list(dev) + for partition in partitions: + try: + info = subprocess.check_output(['sgdisk', '-i', partition.number, + dev]) + info = info.split("\n") # IGNORE:E1103 + for line in info: + for ptype in CEPH_PARTITIONS: + sig = 'Partition GUID code: {}'.format(ptype) + if line.startswith(sig): + return True + except subprocess.CalledProcessError as e: + log("sgdisk inspection of partition {} on {} failed with " + "error: {}. Skipping".format(partition.minor, dev, e.message), + level=ERROR) + return False + + +def start_osds(devices): + # Scan for ceph block devices + rescan_osd_devices() + if cmp_pkgrevno('ceph', "0.56.6") >= 0: + # Use ceph-disk activate for directory based OSD's + for dev_or_path in devices: + if os.path.exists(dev_or_path) and os.path.isdir(dev_or_path): + subprocess.check_call(['ceph-disk', 'activate', dev_or_path]) + + +def rescan_osd_devices(): + cmd = [ + 'udevadm', 'trigger', + '--subsystem-match=block', '--action=add' + ] + + subprocess.call(cmd) + + +_bootstrap_keyring = "/var/lib/ceph/bootstrap-osd/ceph.keyring" +_upgrade_keyring = "/var/lib/ceph/osd/ceph.client.osd-upgrade.keyring" + + +def is_bootstrapped(): + return os.path.exists(_bootstrap_keyring) + + +def wait_for_bootstrap(): + while not is_bootstrapped(): + time.sleep(3) + + +def import_osd_bootstrap_key(key): + if not os.path.exists(_bootstrap_keyring): + cmd = [ + "sudo", + "-u", + ceph_user(), + 'ceph-authtool', + _bootstrap_keyring, + '--create-keyring', + '--name=client.bootstrap-osd', + '--add-key={}'.format(key) + ] + subprocess.check_call(cmd) + + +def import_osd_upgrade_key(key): + if not os.path.exists(_upgrade_keyring): + cmd = [ + "sudo", + "-u", + ceph_user(), + 'ceph-authtool', + _upgrade_keyring, + '--create-keyring', + '--name=client.osd-upgrade', + '--add-key={}'.format(key) + ] + subprocess.check_call(cmd) + + +def generate_monitor_secret(): + cmd = [ + 'ceph-authtool', + '/dev/stdout', + '--name=mon.', + '--gen-key' + ] + res = subprocess.check_output(cmd) + + return "{}==".format(res.split('=')[1].strip()) + +# OSD caps taken from ceph-create-keys +_osd_bootstrap_caps = { + 'mon': [ + 'allow command osd create ...', + 'allow command osd crush set ...', + r'allow command auth add * osd allow\ * mon allow\ rwx', + 'allow command mon getmap' + ] +} + +_osd_bootstrap_caps_profile = { + 'mon': [ + 'allow profile bootstrap-osd' + ] +} + + +def parse_key(raw_key): + # get-or-create appears to have different output depending + # on whether its 'get' or 'create' + # 'create' just returns the key, 'get' is more verbose and + # needs parsing + key = None + if len(raw_key.splitlines()) == 1: + key = raw_key + else: + for element in raw_key.splitlines(): + if 'key' in element: + return element.split(' = ')[1].strip() # IGNORE:E1103 + return key + + +def get_osd_bootstrap_key(): + try: + # Attempt to get/create a key using the OSD bootstrap profile first + key = get_named_key('bootstrap-osd', + _osd_bootstrap_caps_profile) + except: + # If that fails try with the older style permissions + key = get_named_key('bootstrap-osd', + _osd_bootstrap_caps) + return key + + +_radosgw_keyring = "/etc/ceph/keyring.rados.gateway" + + +def import_radosgw_key(key): + if not os.path.exists(_radosgw_keyring): + cmd = [ + "sudo", + "-u", + ceph_user(), + 'ceph-authtool', + _radosgw_keyring, + '--create-keyring', + '--name=client.radosgw.gateway', + '--add-key={}'.format(key) + ] + subprocess.check_call(cmd) + +# OSD caps taken from ceph-create-keys +_radosgw_caps = { + 'mon': ['allow rw'], + 'osd': ['allow rwx'] +} +_upgrade_caps = { + 'mon': ['allow rwx'] +} + + +def get_radosgw_key(pool_list=None): + return get_named_key(name='radosgw.gateway', + caps=_radosgw_caps, + pool_list=pool_list) + + +def get_mds_key(name): + return create_named_keyring(entity='mds', + name=name, + caps=mds_caps) + + +_mds_bootstrap_caps_profile = { + 'mon': [ + 'allow profile bootstrap-mds' + ] +} + + +def get_mds_bootstrap_key(): + return get_named_key('bootstrap-mds', + _mds_bootstrap_caps_profile) + + +_default_caps = collections.OrderedDict([ + ('mon', ['allow r']), + ('osd', ['allow rwx']), +]) + +admin_caps = collections.OrderedDict([ + ('mds', ['allow *']), + ('mon', ['allow *']), + ('osd', ['allow *']) +]) + +mds_caps = collections.OrderedDict([ + ('osd', ['allow *']), + ('mds', ['allow']), + ('mon', ['allow rwx']), +]) + +osd_upgrade_caps = collections.OrderedDict([ + ('mon', ['allow command "config-key"', + 'allow command "osd tree"', + 'allow command "config-key list"', + 'allow command "config-key put"', + 'allow command "config-key get"', + 'allow command "config-key exists"', + 'allow command "osd out"', + 'allow command "osd in"', + 'allow command "osd rm"', + 'allow command "auth del"', + ]) +]) + + +def create_named_keyring(entity, name, caps=None): + caps = caps or _default_caps + cmd = [ + "sudo", + "-u", + ceph_user(), + 'ceph', + '--name', 'mon.', + '--keyring', + '/var/lib/ceph/mon/ceph-{}/keyring'.format( + socket.gethostname() + ), + 'auth', 'get-or-create', '{entity}.{name}'.format(entity=entity, + name=name), + ] + for subsystem, subcaps in caps.items(): + cmd.extend([subsystem, '; '.join(subcaps)]) + log("Calling check_output: {}".format(cmd), level=DEBUG) + return parse_key(subprocess.check_output(cmd).strip()) # IGNORE:E1103 + + +def get_upgrade_key(): + return get_named_key('upgrade-osd', _upgrade_caps) + + +def get_named_key(name, caps=None, pool_list=None): + """Retrieve a specific named cephx key. + + :param name: String Name of key to get. + :param pool_list: The list of pools to give access to + :param caps: dict of cephx capabilities + :returns: Returns a cephx key + """ + try: + # Does the key already exist? + output = subprocess.check_output( + [ + 'sudo', + '-u', ceph_user(), + 'ceph', + '--name', 'mon.', + '--keyring', + '/var/lib/ceph/mon/ceph-{}/keyring'.format( + socket.gethostname() + ), + 'auth', + 'get', + 'client.{}'.format(name), + ]).strip() + return parse_key(output) + except subprocess.CalledProcessError: + # Couldn't get the key, time to create it! + log("Creating new key for {}".format(name), level=DEBUG) + caps = caps or _default_caps + cmd = [ + "sudo", + "-u", + ceph_user(), + 'ceph', + '--name', 'mon.', + '--keyring', + '/var/lib/ceph/mon/ceph-{}/keyring'.format( + socket.gethostname() + ), + 'auth', 'get-or-create', 'client.{}'.format(name), + ] + # Add capabilities + for subsystem, subcaps in caps.items(): + if subsystem == 'osd': + if pool_list: + # This will output a string similar to: + # "pool=rgw pool=rbd pool=something" + pools = " ".join(['pool={0}'.format(i) for i in pool_list]) + subcaps[0] = subcaps[0] + " " + pools + cmd.extend([subsystem, '; '.join(subcaps)]) + + log("Calling check_output: {}".format(cmd), level=DEBUG) + return parse_key(subprocess.check_output(cmd).strip()) # IGNORE:E1103 + + +def upgrade_key_caps(key, caps): + """ Upgrade key to have capabilities caps """ + if not is_leader(): + # Not the MON leader OR not clustered + return + cmd = [ + "sudo", "-u", ceph_user(), 'ceph', 'auth', 'caps', key + ] + for subsystem, subcaps in caps.items(): + cmd.extend([subsystem, '; '.join(subcaps)]) + subprocess.check_call(cmd) + + +@cached +def systemd(): + return CompareHostReleases(lsb_release()['DISTRIB_CODENAME']) >= 'vivid' + + +def bootstrap_monitor_cluster(secret): + hostname = socket.gethostname() + path = '/var/lib/ceph/mon/ceph-{}'.format(hostname) + done = '{}/done'.format(path) + if systemd(): + init_marker = '{}/systemd'.format(path) + else: + init_marker = '{}/upstart'.format(path) + + keyring = '/var/lib/ceph/tmp/{}.mon.keyring'.format(hostname) + + if os.path.exists(done): + log('bootstrap_monitor_cluster: mon already initialized.') + else: + # Ceph >= 0.61.3 needs this for ceph-mon fs creation + mkdir('/var/run/ceph', owner=ceph_user(), + group=ceph_user(), perms=0o755) + mkdir(path, owner=ceph_user(), group=ceph_user()) + # end changes for Ceph >= 0.61.3 + try: + subprocess.check_call(['ceph-authtool', keyring, + '--create-keyring', '--name=mon.', + '--add-key={}'.format(secret), + '--cap', 'mon', 'allow *']) + + subprocess.check_call(['ceph-mon', '--mkfs', + '-i', hostname, + '--keyring', keyring]) + chownr(path, ceph_user(), ceph_user()) + with open(done, 'w'): + pass + with open(init_marker, 'w'): + pass + + if systemd(): + subprocess.check_call(['systemctl', 'enable', 'ceph-mon']) + service_restart('ceph-mon') + else: + service_restart('ceph-mon-all') + + if cmp_pkgrevno('ceph', '12.0.0') >= 0: + # NOTE(jamespage): Later ceph releases require explicit + # call to ceph-create-keys to setup the + # admin keys for the cluster; this command + # will wait for quorum in the cluster before + # returning. + cmd = ['ceph-create-keys', '--id', hostname] + subprocess.check_call(cmd) + except: + raise + finally: + os.unlink(keyring) + + +def update_monfs(): + hostname = socket.gethostname() + monfs = '/var/lib/ceph/mon/ceph-{}'.format(hostname) + if systemd(): + init_marker = '{}/systemd'.format(monfs) + else: + init_marker = '{}/upstart'.format(monfs) + if os.path.exists(monfs) and not os.path.exists(init_marker): + # Mark mon as managed by upstart so that + # it gets start correctly on reboots + with open(init_marker, 'w'): + pass + + +def maybe_zap_journal(journal_dev): + if is_osd_disk(journal_dev): + log('Looks like {} is already an OSD data' + ' or journal, skipping.'.format(journal_dev)) + return + zap_disk(journal_dev) + log("Zapped journal device {}".format(journal_dev)) + + +def get_partitions(dev): + cmd = ['partx', '--raw', '--noheadings', dev] + try: + out = subprocess.check_output(cmd).splitlines() + log("get partitions: {}".format(out), level=DEBUG) + return out + except subprocess.CalledProcessError as e: + log("Can't get info for {0}: {1}".format(dev, e.output)) + return [] + + +def find_least_used_journal(journal_devices): + usages = map(lambda a: (len(get_partitions(a)), a), journal_devices) + least = min(usages, key=lambda t: t[0]) + return least[1] + + +def osdize(dev, osd_format, osd_journal, reformat_osd=False, + ignore_errors=False, encrypt=False, bluestore=False): + if dev.startswith('/dev'): + osdize_dev(dev, osd_format, osd_journal, + reformat_osd, ignore_errors, encrypt, + bluestore) + else: + osdize_dir(dev, encrypt) + + +def osdize_dev(dev, osd_format, osd_journal, reformat_osd=False, + ignore_errors=False, encrypt=False, bluestore=False): + if not os.path.exists(dev): + log('Path {} does not exist - bailing'.format(dev)) + return + + if not is_block_device(dev): + log('Path {} is not a block device - bailing'.format(dev)) + return + + if is_osd_disk(dev) and not reformat_osd: + log('Looks like {} is already an' + ' OSD data or journal, skipping.'.format(dev)) + return + + if is_device_mounted(dev): + log('Looks like {} is in use, skipping.'.format(dev)) + return + + status_set('maintenance', 'Initializing device {}'.format(dev)) + cmd = ['ceph-disk', 'prepare'] + # Later versions of ceph support more options + if cmp_pkgrevno('ceph', '0.60') >= 0: + if encrypt: + cmd.append('--dmcrypt') + if cmp_pkgrevno('ceph', '0.48.3') >= 0: + if osd_format: + cmd.append('--fs-type') + cmd.append(osd_format) + + if reformat_osd: + cmd.append('--zap-disk') + + # NOTE(jamespage): enable experimental bluestore support + if cmp_pkgrevno('ceph', '10.2.0') >= 0 and bluestore: + cmd.append('--bluestore') + + cmd.append(dev) + + if osd_journal: + least_used = find_least_used_journal(osd_journal) + cmd.append(least_used) + else: + # Just provide the device - no other options + # for older versions of ceph + cmd.append(dev) + if reformat_osd: + zap_disk(dev) + + try: + log("osdize cmd: {}".format(cmd)) + subprocess.check_call(cmd) + except subprocess.CalledProcessError: + if ignore_errors: + log('Unable to initialize device: {}'.format(dev), WARNING) + else: + log('Unable to initialize device: {}'.format(dev), ERROR) + raise + + +def osdize_dir(path, encrypt=False): + """Ask ceph-disk to prepare a directory to become an osd. + + :param path: str. The directory to osdize + :param encrypt: bool. Should the OSD directory be encrypted at rest + :returns: None + """ + if os.path.exists(os.path.join(path, 'upstart')): + log('Path {} is already configured as an OSD - bailing'.format(path)) + return + + if cmp_pkgrevno('ceph', "0.56.6") < 0: + log('Unable to use directories for OSDs with ceph < 0.56.6', + level=ERROR) + return + + mkdir(path, owner=ceph_user(), group=ceph_user(), perms=0o755) + chownr('/var/lib/ceph', ceph_user(), ceph_user()) + cmd = [ + 'sudo', '-u', ceph_user(), + 'ceph-disk', + 'prepare', + '--data-dir', + path + ] + if cmp_pkgrevno('ceph', '0.60') >= 0: + if encrypt: + cmd.append('--dmcrypt') + log("osdize dir cmd: {}".format(cmd)) + subprocess.check_call(cmd) + + +def filesystem_mounted(fs): + return subprocess.call(['grep', '-wqs', fs, '/proc/mounts']) == 0 + + +def get_running_osds(): + """Returns a list of the pids of the current running OSD daemons""" + cmd = ['pgrep', 'ceph-osd'] + try: + result = subprocess.check_output(cmd) + return result.split() + except subprocess.CalledProcessError: + return [] + + +def get_cephfs(service): + """List the Ceph Filesystems that exist. + + :param service: The service name to run the ceph command under + :returns: list. Returns a list of the ceph filesystems + """ + if get_version() < 0.86: + # This command wasn't introduced until 0.86 ceph + return [] + try: + output = subprocess.check_output(["ceph", '--id', service, "fs", "ls"]) + if not output: + return [] + """ + Example subprocess output: + 'name: ip-172-31-23-165, metadata pool: ip-172-31-23-165_metadata, + data pools: [ip-172-31-23-165_data ]\n' + output: filesystems: ['ip-172-31-23-165'] + """ + filesystems = [] + for line in output.splitlines(): + parts = line.split(',') + for part in parts: + if "name" in part: + filesystems.append(part.split(' ')[1]) + except subprocess.CalledProcessError: + return [] + + +def wait_for_all_monitors_to_upgrade(new_version, upgrade_key): + """Fairly self explanatory name. This function will wait + for all monitors in the cluster to upgrade or it will + return after a timeout period has expired. + + :param new_version: str of the version to watch + :param upgrade_key: the cephx key name to use + """ + done = False + start_time = time.time() + monitor_list = [] + + mon_map = get_mon_map('admin') + if mon_map['monmap']['mons']: + for mon in mon_map['monmap']['mons']: + monitor_list.append(mon['name']) + while not done: + try: + done = all(monitor_key_exists(upgrade_key, "{}_{}_{}_done".format( + "mon", mon, new_version + )) for mon in monitor_list) + current_time = time.time() + if current_time > (start_time + 10 * 60): + raise Exception + else: + # Wait 30 seconds and test again if all monitors are upgraded + time.sleep(30) + except subprocess.CalledProcessError: + raise + + +# Edge cases: +# 1. Previous node dies on upgrade, can we retry? +def roll_monitor_cluster(new_version, upgrade_key): + """This is tricky to get right so here's what we're going to do. + + There's 2 possible cases: Either I'm first in line or not. + If I'm not first in line I'll wait a random time between 5-30 seconds + and test to see if the previous monitor is upgraded yet. + + :param new_version: str of the version to upgrade to + :param upgrade_key: the cephx key name to use when upgrading + """ + log('roll_monitor_cluster called with {}'.format(new_version)) + my_name = socket.gethostname() + monitor_list = [] + mon_map = get_mon_map('admin') + if mon_map['monmap']['mons']: + for mon in mon_map['monmap']['mons']: + monitor_list.append(mon['name']) + else: + status_set('blocked', 'Unable to get monitor cluster information') + sys.exit(1) + log('monitor_list: {}'.format(monitor_list)) + + # A sorted list of osd unit names + mon_sorted_list = sorted(monitor_list) + + try: + position = mon_sorted_list.index(my_name) + log("upgrade position: {}".format(position)) + if position == 0: + # I'm first! Roll + # First set a key to inform others I'm about to roll + lock_and_roll(upgrade_key=upgrade_key, + service='mon', + my_name=my_name, + version=new_version) + else: + # Check if the previous node has finished + status_set('waiting', + 'Waiting on {} to finish upgrading'.format( + mon_sorted_list[position - 1])) + wait_on_previous_node(upgrade_key=upgrade_key, + service='mon', + previous_node=mon_sorted_list[position - 1], + version=new_version) + lock_and_roll(upgrade_key=upgrade_key, + service='mon', + my_name=my_name, + version=new_version) + except ValueError: + log("Failed to find {} in list {}.".format( + my_name, mon_sorted_list)) + status_set('blocked', 'failed to upgrade monitor') + + +def upgrade_monitor(new_version): + """Upgrade the current ceph monitor to the new version + + :param new_version: String version to upgrade to. + """ + current_version = get_version() + status_set("maintenance", "Upgrading monitor") + log("Current ceph version is {}".format(current_version)) + log("Upgrading to: {}".format(new_version)) + + try: + add_source(config('source'), config('key')) + apt_update(fatal=True) + except subprocess.CalledProcessError as err: + log("Adding the ceph source failed with message: {}".format( + err.message)) + status_set("blocked", "Upgrade to {} failed".format(new_version)) + sys.exit(1) + try: + if systemd(): + for mon_id in get_local_mon_ids(): + service_stop('ceph-mon@{}'.format(mon_id)) + else: + service_stop('ceph-mon-all') + apt_install(packages=determine_packages(), fatal=True) + + # Ensure the files and directories under /var/lib/ceph is chowned + # properly as part of the move to the Jewel release, which moved the + # ceph daemons to running as ceph:ceph instead of root:root. + if new_version == 'jewel': + # Ensure the ownership of Ceph's directories is correct + owner = ceph_user() + chownr(path=os.path.join(os.sep, "var", "lib", "ceph"), + owner=owner, + group=owner, + follow_links=True) + + if systemd(): + for mon_id in get_local_mon_ids(): + service_start('ceph-mon@{}'.format(mon_id)) + else: + service_start('ceph-mon-all') + except subprocess.CalledProcessError as err: + log("Stopping ceph and upgrading packages failed " + "with message: {}".format(err.message)) + status_set("blocked", "Upgrade to {} failed".format(new_version)) + sys.exit(1) + + +def lock_and_roll(upgrade_key, service, my_name, version): + """Create a lock on the ceph monitor cluster and upgrade. + + :param upgrade_key: str. The cephx key to use + :param service: str. The cephx id to use + :param my_name: str. The current hostname + :param version: str. The version we are upgrading to + """ + start_timestamp = time.time() + + log('monitor_key_set {}_{}_{}_start {}'.format( + service, + my_name, + version, + start_timestamp)) + monitor_key_set(upgrade_key, "{}_{}_{}_start".format( + service, my_name, version), start_timestamp) + log("Rolling") + + # This should be quick + if service == 'osd': + upgrade_osd(version) + elif service == 'mon': + upgrade_monitor(version) + else: + log("Unknown service {}. Unable to upgrade".format(service), + level=ERROR) + log("Done") + + stop_timestamp = time.time() + # Set a key to inform others I am finished + log('monitor_key_set {}_{}_{}_done {}'.format(service, + my_name, + version, + stop_timestamp)) + status_set('maintenance', 'Finishing upgrade') + monitor_key_set(upgrade_key, "{}_{}_{}_done".format(service, + my_name, + version), + stop_timestamp) + + +def wait_on_previous_node(upgrade_key, service, previous_node, version): + """A lock that sleeps the current thread while waiting for the previous + node to finish upgrading. + + :param upgrade_key: + :param service: str. the cephx id to use + :param previous_node: str. The name of the previous node to wait on + :param version: str. The version we are upgrading to + :returns: None + """ + log("Previous node is: {}".format(previous_node)) + + previous_node_finished = monitor_key_exists( + upgrade_key, + "{}_{}_{}_done".format(service, previous_node, version)) + + while previous_node_finished is False: + log("{} is not finished. Waiting".format(previous_node)) + # Has this node been trying to upgrade for longer than + # 10 minutes? + # If so then move on and consider that node dead. + + # NOTE: This assumes the clusters clocks are somewhat accurate + # If the hosts clock is really far off it may cause it to skip + # the previous node even though it shouldn't. + current_timestamp = time.time() + previous_node_start_time = monitor_key_get( + upgrade_key, + "{}_{}_{}_start".format(service, previous_node, version)) + if (current_timestamp - (10 * 60)) > previous_node_start_time: + # Previous node is probably dead. Lets move on + if previous_node_start_time is not None: + log( + "Waited 10 mins on node {}. current time: {} > " + "previous node start time: {} Moving on".format( + previous_node, + (current_timestamp - (10 * 60)), + previous_node_start_time)) + return + else: + # I have to wait. Sleep a random amount of time and then + # check if I can lock,upgrade and roll. + wait_time = random.randrange(5, 30) + log('waiting for {} seconds'.format(wait_time)) + time.sleep(wait_time) + previous_node_finished = monitor_key_exists( + upgrade_key, + "{}_{}_{}_done".format(service, previous_node, version)) + + +def get_upgrade_position(osd_sorted_list, match_name): + """Return the upgrade position for the given osd. + + :param osd_sorted_list: list. Osds sorted + :param match_name: str. The osd name to match + :returns: int. The position or None if not found + """ + for index, item in enumerate(osd_sorted_list): + if item.name == match_name: + return index + return None + + +# Edge cases: +# 1. Previous node dies on upgrade, can we retry? +# 2. This assumes that the osd failure domain is not set to osd. +# It rolls an entire server at a time. +def roll_osd_cluster(new_version, upgrade_key): + """This is tricky to get right so here's what we're going to do. + + There's 2 possible cases: Either I'm first in line or not. + If I'm not first in line I'll wait a random time between 5-30 seconds + and test to see if the previous osd is upgraded yet. + + TODO: If you're not in the same failure domain it's safe to upgrade + 1. Examine all pools and adopt the most strict failure domain policy + Example: Pool 1: Failure domain = rack + Pool 2: Failure domain = host + Pool 3: Failure domain = row + + outcome: Failure domain = host + + :param new_version: str of the version to upgrade to + :param upgrade_key: the cephx key name to use when upgrading + """ + log('roll_osd_cluster called with {}'.format(new_version)) + my_name = socket.gethostname() + osd_tree = get_osd_tree(service=upgrade_key) + # A sorted list of osd unit names + osd_sorted_list = sorted(osd_tree) + log("osd_sorted_list: {}".format(osd_sorted_list)) + + try: + position = get_upgrade_position(osd_sorted_list, my_name) + log("upgrade position: {}".format(position)) + if position == 0: + # I'm first! Roll + # First set a key to inform others I'm about to roll + lock_and_roll(upgrade_key=upgrade_key, + service='osd', + my_name=my_name, + version=new_version) + else: + # Check if the previous node has finished + status_set('blocked', + 'Waiting on {} to finish upgrading'.format( + osd_sorted_list[position - 1].name)) + wait_on_previous_node( + upgrade_key=upgrade_key, + service='osd', + previous_node=osd_sorted_list[position - 1].name, + version=new_version) + lock_and_roll(upgrade_key=upgrade_key, + service='osd', + my_name=my_name, + version=new_version) + except ValueError: + log("Failed to find name {} in list {}".format( + my_name, osd_sorted_list)) + status_set('blocked', 'failed to upgrade osd') + + +def upgrade_osd(new_version): + """Upgrades the current osd + + :param new_version: str. The new version to upgrade to + """ + current_version = get_version() + status_set("maintenance", "Upgrading osd") + log("Current ceph version is {}".format(current_version)) + log("Upgrading to: {}".format(new_version)) + + try: + add_source(config('source'), config('key')) + apt_update(fatal=True) + except subprocess.CalledProcessError as err: + log("Adding the ceph sources failed with message: {}".format( + err.message)) + status_set("blocked", "Upgrade to {} failed".format(new_version)) + sys.exit(1) + + try: + # Upgrade the packages before restarting the daemons. + status_set('maintenance', 'Upgrading packages to %s' % new_version) + apt_install(packages=determine_packages(), fatal=True) + + # If the upgrade does not need an ownership update of any of the + # directories in the osd service directory, then simply restart + # all of the OSDs at the same time as this will be the fastest + # way to update the code on the node. + if not dirs_need_ownership_update('osd'): + log('Restarting all OSDs to load new binaries', DEBUG) + service_restart('ceph-osd-all') + return + + # Need to change the ownership of all directories which are not OSD + # directories as well. + # TODO - this should probably be moved to the general upgrade function + # and done before mon/osd. + update_owner(CEPH_BASE_DIR, recurse_dirs=False) + non_osd_dirs = filter(lambda x: not x == 'osd', + os.listdir(CEPH_BASE_DIR)) + non_osd_dirs = map(lambda x: os.path.join(CEPH_BASE_DIR, x), + non_osd_dirs) + for path in non_osd_dirs: + update_owner(path) + + # Fast service restart wasn't an option because each of the OSD + # directories need the ownership updated for all the files on + # the OSD. Walk through the OSDs one-by-one upgrading the OSD. + for osd_dir in _get_child_dirs(OSD_BASE_DIR): + try: + osd_num = _get_osd_num_from_dirname(osd_dir) + _upgrade_single_osd(osd_num, osd_dir) + except ValueError as ex: + # Directory could not be parsed - junk directory? + log('Could not parse osd directory %s: %s' % (osd_dir, ex), + WARNING) + continue + + except (subprocess.CalledProcessError, IOError) as err: + log("Stopping ceph and upgrading packages failed " + "with message: {}".format(err.message)) + status_set("blocked", "Upgrade to {} failed".format(new_version)) + sys.exit(1) + + +def _upgrade_single_osd(osd_num, osd_dir): + """Upgrades the single OSD directory. + + :param osd_num: the num of the OSD + :param osd_dir: the directory of the OSD to upgrade + :raises CalledProcessError: if an error occurs in a command issued as part + of the upgrade process + :raises IOError: if an error occurs reading/writing to a file as part + of the upgrade process + """ + stop_osd(osd_num) + disable_osd(osd_num) + update_owner(osd_dir) + enable_osd(osd_num) + start_osd(osd_num) + + +def stop_osd(osd_num): + """Stops the specified OSD number. + + :param osd_num: the osd number to stop + """ + if systemd(): + service_stop('ceph-osd@{}'.format(osd_num)) + else: + service_stop('ceph-osd', id=osd_num) + + +def start_osd(osd_num): + """Starts the specified OSD number. + + :param osd_num: the osd number to start. + """ + if systemd(): + service_start('ceph-osd@{}'.format(osd_num)) + else: + service_start('ceph-osd', id=osd_num) + + +def disable_osd(osd_num): + """Disables the specified OSD number. + + Ensures that the specified osd will not be automatically started at the + next reboot of the system. Due to differences between init systems, + this method cannot make any guarantees that the specified osd cannot be + started manually. + + :param osd_num: the osd id which should be disabled. + :raises CalledProcessError: if an error occurs invoking the systemd cmd + to disable the OSD + :raises IOError, OSError: if the attempt to read/remove the ready file in + an upstart enabled system fails + """ + if systemd(): + # When running under systemd, the individual ceph-osd daemons run as + # templated units and can be directly addressed by referring to the + # templated service name ceph-osd@. Additionally, systemd + # allows one to disable a specific templated unit by running the + # 'systemctl disable ceph-osd@' command. When disabled, the + # OSD should remain disabled until re-enabled via systemd. + # Note: disabling an already disabled service in systemd returns 0, so + # no need to check whether it is enabled or not. + cmd = ['systemctl', 'disable', 'ceph-osd@{}'.format(osd_num)] + subprocess.check_call(cmd) + else: + # Neither upstart nor the ceph-osd upstart script provides for + # disabling the starting of an OSD automatically. The specific OSD + # cannot be prevented from running manually, however it can be + # prevented from running automatically on reboot by removing the + # 'ready' file in the OSD's root directory. This is due to the + # ceph-osd-all upstart script checking for the presence of this file + # before starting the OSD. + ready_file = os.path.join(OSD_BASE_DIR, 'ceph-{}'.format(osd_num), + 'ready') + if os.path.exists(ready_file): + os.unlink(ready_file) + + +def enable_osd(osd_num): + """Enables the specified OSD number. + + Ensures that the specified osd_num will be enabled and ready to start + automatically in the event of a reboot. + + :param osd_num: the osd id which should be enabled. + :raises CalledProcessError: if the call to the systemd command issued + fails when enabling the service + :raises IOError: if the attempt to write the ready file in an usptart + enabled system fails + """ + if systemd(): + cmd = ['systemctl', 'enable', 'ceph-osd@{}'.format(osd_num)] + subprocess.check_call(cmd) + else: + # When running on upstart, the OSDs are started via the ceph-osd-all + # upstart script which will only start the osd if it has a 'ready' + # file. Make sure that file exists. + ready_file = os.path.join(OSD_BASE_DIR, 'ceph-{}'.format(osd_num), + 'ready') + with open(ready_file, 'w') as f: + f.write('ready') + + # Make sure the correct user owns the file. It shouldn't be necessary + # as the upstart script should run with root privileges, but its better + # to have all the files matching ownership. + update_owner(ready_file) + + +def update_owner(path, recurse_dirs=True): + """Changes the ownership of the specified path. + + Changes the ownership of the specified path to the new ceph daemon user + using the system's native chown functionality. This may take awhile, + so this method will issue a set_status for any changes of ownership which + recurses into directory structures. + + :param path: the path to recursively change ownership for + :param recurse_dirs: boolean indicating whether to recursively change the + ownership of all the files in a path's subtree or to + simply change the ownership of the path. + :raises CalledProcessError: if an error occurs issuing the chown system + command + """ + user = ceph_user() + user_group = '{ceph_user}:{ceph_user}'.format(ceph_user=user) + cmd = ['chown', user_group, path] + if os.path.isdir(path) and recurse_dirs: + status_set('maintenance', ('Updating ownership of %s to %s' % + (path, user))) + cmd.insert(1, '-R') + + log('Changing ownership of {path} to {user}'.format( + path=path, user=user_group), DEBUG) + start = datetime.now() + subprocess.check_call(cmd) + elapsed_time = (datetime.now() - start) + + log('Took {secs} seconds to change the ownership of path: {path}'.format( + secs=elapsed_time.total_seconds(), path=path), DEBUG) + + +def list_pools(service): + """This will list the current pools that Ceph has + + :param service: String service id to run under + :returns: list. Returns a list of the ceph pools. + :raises: CalledProcessError if the subprocess fails to run. + """ + try: + pool_list = [] + pools = subprocess.check_output(['rados', '--id', service, 'lspools']) + for pool in pools.splitlines(): + pool_list.append(pool) + return pool_list + except subprocess.CalledProcessError as err: + log("rados lspools failed with error: {}".format(err.output)) + raise + + +def dirs_need_ownership_update(service): + """Determines if directories still need change of ownership. + + Examines the set of directories under the /var/lib/ceph/{service} directory + and determines if they have the correct ownership or not. This is + necessary due to the upgrade from Hammer to Jewel where the daemon user + changes from root: to ceph:. + + :param service: the name of the service folder to check (e.g. osd, mon) + :returns: boolean. True if the directories need a change of ownership, + False otherwise. + :raises IOError: if an error occurs reading the file stats from one of + the child directories. + :raises OSError: if the specified path does not exist or some other error + """ + expected_owner = expected_group = ceph_user() + path = os.path.join(CEPH_BASE_DIR, service) + for child in _get_child_dirs(path): + curr_owner, curr_group = owner(child) + + if (curr_owner == expected_owner) and (curr_group == expected_group): + continue + + log('Directory "%s" needs its ownership updated' % child, DEBUG) + return True + + # All child directories had the expected ownership + return False + +# A dict of valid ceph upgrade paths. Mapping is old -> new +UPGRADE_PATHS = { + 'firefly': 'hammer', + 'hammer': 'jewel', +} + +# Map UCA codenames to ceph codenames +UCA_CODENAME_MAP = { + 'icehouse': 'firefly', + 'juno': 'firefly', + 'kilo': 'hammer', + 'liberty': 'hammer', + 'mitaka': 'jewel', + 'newton': 'jewel', + 'ocata': 'jewel', +} + + +def pretty_print_upgrade_paths(): + """Pretty print supported upgrade paths for ceph""" + lines = [] + for key, value in UPGRADE_PATHS.iteritems(): + lines.append("{} -> {}".format(key, value)) + return lines + + +def resolve_ceph_version(source): + """Resolves a version of ceph based on source configuration + based on Ubuntu Cloud Archive pockets. + + @param: source: source configuration option of charm + :returns: ceph release codename or None if not resolvable + """ + os_release = get_os_codename_install_source(source) + return UCA_CODENAME_MAP.get(os_release) + + +def get_ceph_pg_stat(): + """Returns the result of ceph pg stat. + + :returns: dict + """ + try: + tree = subprocess.check_output(['ceph', 'pg', 'stat', '--format=json']) + try: + json_tree = json.loads(tree) + if not json_tree['num_pg_by_state']: + return None + return json_tree + except ValueError as v: + log("Unable to parse ceph pg stat json: {}. Error: {}".format( + tree, v.message)) + raise + except subprocess.CalledProcessError as e: + log("ceph pg stat command failed with message: {}".format( + e.message)) + raise + + +def get_ceph_health(): + """Returns the health of the cluster from a 'ceph status' + + :returns: dict tree of ceph status + :raises: CalledProcessError if our ceph command fails to get the overall + status, use get_ceph_health()['overall_status']. + """ + try: + tree = subprocess.check_output( + ['ceph', 'status', '--format=json']) + try: + json_tree = json.loads(tree) + # Make sure children are present in the json + if not json_tree['overall_status']: + return None + + return json_tree + except ValueError as v: + log("Unable to parse ceph tree json: {}. Error: {}".format( + tree, v.message)) + raise + except subprocess.CalledProcessError as e: + log("ceph status command failed with message: {}".format( + e.message)) + raise + + +def reweight_osd(osd_num, new_weight): + """Changes the crush weight of an OSD to the value specified. + + :param osd_num: the osd id which should be changed + :param new_weight: the new weight for the OSD + :returns: bool. True if output looks right, else false. + :raises CalledProcessError: if an error occurs invoking the systemd cmd + """ + try: + cmd_result = subprocess.check_output( + ['ceph', 'osd', 'crush', 'reweight', "osd.{}".format(osd_num), + new_weight], stderr=subprocess.STDOUT) + expected_result = "reweighted item id {ID} name \'osd.{ID}\'".format( + ID=osd_num) + " to {}".format(new_weight) + log(cmd_result) + if expected_result in cmd_result: + return True + return False + except subprocess.CalledProcessError as e: + log("ceph osd crush reweight command failed with message: {}".format( + e.message)) + raise + + +def determine_packages(): + """Determines packages for installation. + + :returns: list of ceph packages + """ + if is_container(): + PACKAGES.remove('ntp') + + return PACKAGES + + +def bootstrap_manager(): + hostname = socket.gethostname() + path = '/var/lib/ceph/mgr/ceph-{}'.format(hostname) + keyring = os.path.join(path, 'keyring') + + if os.path.exists(keyring): + log('bootstrap_manager: mgr already initialized.') + else: + mkdir(path, owner=ceph_user(), group=ceph_user()) + subprocess.check_call(['ceph', 'auth', 'get-or-create', + 'mgr.{}'.format(hostname), 'mon', + 'allow profile mgr', 'osd', 'allow *', + 'mds', 'allow *', '--out-file', + keyring]) + chownr(path, ceph_user(), ceph_user()) + + unit = 'ceph-mgr@{}'.format(hostname) + subprocess.check_call(['systemctl', 'enable', unit]) + service_restart(unit) diff --git a/lib/setup.py b/lib/setup.py deleted file mode 100644 index 139c80d..0000000 --- a/lib/setup.py +++ /dev/null @@ -1,85 +0,0 @@ -# -*- coding: utf-8 -*- -from __future__ import print_function - -import os -import sys -from setuptools import setup, find_packages -from setuptools.command.test import test as TestCommand - -version = "0.0.1.dev1" -install_require = [ -] - -tests_require = [ - 'tox >= 2.3.1', -] - - -class Tox(TestCommand): - - user_options = [('tox-args=', 'a', "Arguments to pass to tox")] - - def initialize_options(self): - TestCommand.initialize_options(self) - self.tox_args = None - - def finalize_options(self): - TestCommand.finalize_options(self) - self.test_args = [] - self.test_suite = True - - def run_tests(self): - # import here, cause outside the eggs aren't loaded - import tox - import shlex - args = self.tox_args - # remove the 'test' arg from argv as tox passes it to ostestr which - # breaks it. - sys.argv.pop() - if args: - args = shlex.split(self.tox_args) - errno = tox.cmdline(args=args) - sys.exit(errno) - - -if sys.argv[-1] == 'publish': - os.system("python setup.py sdist upload") - os.system("python setup.py bdist_wheel upload") - sys.exit() - - -if sys.argv[-1] == 'tag': - os.system("git tag -a %s -m 'version %s'" % (version, version)) - os.system("git push --tags") - sys.exit() - - -setup( - name='charms.ceph', - version=version, - description='Provide base module for ceph charms.', - classifiers=[ - "Development Status :: 2 - Pre-Alpha", - "Intended Audience :: Developers", - "Topic :: System", - "Topic :: System :: Installation/Setup", - "Topic :: System :: Software Distribution", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.5", - "License :: OSI Approved :: Apache Software License", - ], - url='https://github.com/openstack/charms.ceph', - author='OpenStack Charmers', - author_email='openstack-dev@lists.openstack.org', - license='Apache-2.0: http://www.apache.org/licenses/LICENSE-2.0', - packages=find_packages(exclude=["unit_tests"]), - zip_safe=False, - cmdclass={'test': Tox}, - install_requires=install_require, - extras_require={ - 'testing': tests_require, - }, - tests_require=tests_require, -) diff --git a/unit_tests/test_ceph_ops.py b/unit_tests/test_ceph_ops.py index 3b82196..5f17e03 100644 --- a/unit_tests/test_ceph_ops.py +++ b/unit_tests/test_ceph_ops.py @@ -20,13 +20,13 @@ from mock import ( patch, ) -from ceph import ceph_broker +from ceph import broker class TestCephOps(unittest.TestCase): - @patch.object(ceph_broker, 'create_erasure_profile') - @patch.object(ceph_broker, 'log', lambda *args, **kwargs: None) + @patch.object(broker, 'create_erasure_profile') + @patch.object(broker, 'log', lambda *args, **kwargs: None) def test_create_erasure_profile(self, mock_create_erasure): req = json.dumps({'api-version': 1, 'ops': [{ @@ -37,7 +37,7 @@ class TestCephOps(unittest.TestCase): 'k': 3, 'm': 2, }]}) - rc = ceph_broker.process_requests(req) + rc = broker.process_requests(req) mock_create_erasure.assert_called_with(service='admin', profile_name='foo', coding_chunks=2, @@ -47,9 +47,9 @@ class TestCephOps(unittest.TestCase): erasure_plugin_name='jerasure') self.assertEqual(json.loads(rc), {'exit-code': 0}) - @patch.object(ceph_broker, 'pool_exists') - @patch.object(ceph_broker, 'ReplicatedPool') - @patch.object(ceph_broker, 'log', lambda *args, **kwargs: None) + @patch.object(broker, 'pool_exists') + @patch.object(broker, 'ReplicatedPool') + @patch.object(broker, 'log', lambda *args, **kwargs: None) def test_process_requests_create_replicated_pool(self, mock_replicated_pool, mock_pool_exists): @@ -61,15 +61,15 @@ class TestCephOps(unittest.TestCase): 'name': 'foo', 'replicas': 3 }]}) - rc = ceph_broker.process_requests(reqs) + rc = broker.process_requests(reqs) mock_pool_exists.assert_called_with(service='admin', name='foo') calls = [call(name=u'foo', service='admin', replicas=3)] mock_replicated_pool.assert_has_calls(calls) self.assertEqual(json.loads(rc), {'exit-code': 0}) - @patch.object(ceph_broker, 'pool_exists') - @patch.object(ceph_broker, 'ReplicatedPool') - @patch.object(ceph_broker, 'log', lambda *args, **kwargs: None) + @patch.object(broker, 'pool_exists') + @patch.object(broker, 'ReplicatedPool') + @patch.object(broker, 'log', lambda *args, **kwargs: None) def test_process_requests_replicated_pool_weight(self, mock_replicated_pool, mock_pool_exists): @@ -82,15 +82,15 @@ class TestCephOps(unittest.TestCase): 'weight': 40.0, 'replicas': 3 }]}) - rc = ceph_broker.process_requests(reqs) + rc = broker.process_requests(reqs) mock_pool_exists.assert_called_with(service='admin', name='foo') calls = [call(name=u'foo', service='admin', replicas=3, percent_data=40.0)] mock_replicated_pool.assert_has_calls(calls) self.assertEqual(json.loads(rc), {'exit-code': 0}) - @patch.object(ceph_broker, 'delete_pool') - @patch.object(ceph_broker, 'log', lambda *args, **kwargs: None) + @patch.object(broker, 'delete_pool') + @patch.object(broker, 'log', lambda *args, **kwargs: None) def test_process_requests_delete_pool(self, mock_delete_pool): reqs = json.dumps({'api-version': 1, @@ -99,14 +99,14 @@ class TestCephOps(unittest.TestCase): 'name': 'foo', }]}) mock_delete_pool.return_value = {'exit-code': 0} - rc = ceph_broker.process_requests(reqs) + rc = broker.process_requests(reqs) mock_delete_pool.assert_called_with(service='admin', name='foo') self.assertEqual(json.loads(rc), {'exit-code': 0}) - @patch.object(ceph_broker, 'pool_exists') - @patch.object(ceph_broker.ErasurePool, 'create') - @patch.object(ceph_broker, 'erasure_profile_exists') - @patch.object(ceph_broker, 'log', lambda *args, **kwargs: None) + @patch.object(broker, 'pool_exists') + @patch.object(broker.ErasurePool, 'create') + @patch.object(broker, 'erasure_profile_exists') + @patch.object(broker, 'log', lambda *args, **kwargs: None) def test_process_requests_create_erasure_pool(self, mock_profile_exists, mock_erasure_pool, mock_pool_exists): @@ -118,15 +118,15 @@ class TestCephOps(unittest.TestCase): 'name': 'foo', 'erasure-profile': 'default' }]}) - rc = ceph_broker.process_requests(reqs) + rc = broker.process_requests(reqs) mock_profile_exists.assert_called_with(service='admin', name='default') mock_pool_exists.assert_called_with(service='admin', name='foo') mock_erasure_pool.assert_called_with() self.assertEqual(json.loads(rc), {'exit-code': 0}) - @patch.object(ceph_broker, 'pool_exists') - @patch.object(ceph_broker.Pool, 'add_cache_tier') - @patch.object(ceph_broker, 'log', lambda *args, **kwargs: None) + @patch.object(broker, 'pool_exists') + @patch.object(broker.Pool, 'add_cache_tier') + @patch.object(broker, 'log', lambda *args, **kwargs: None) def test_process_requests_create_cache_tier(self, mock_pool, mock_pool_exists): mock_pool_exists.return_value = True @@ -138,16 +138,16 @@ class TestCephOps(unittest.TestCase): 'mode': 'writeback', 'erasure-profile': 'default' }]}) - rc = ceph_broker.process_requests(reqs) + rc = broker.process_requests(reqs) mock_pool_exists.assert_any_call(service='admin', name='foo') mock_pool_exists.assert_any_call(service='admin', name='foo-ssd') mock_pool.assert_called_with(cache_pool='foo-ssd', mode='writeback') self.assertEqual(json.loads(rc), {'exit-code': 0}) - @patch.object(ceph_broker, 'pool_exists') - @patch.object(ceph_broker.Pool, 'remove_cache_tier') - @patch.object(ceph_broker, 'log', lambda *args, **kwargs: None) + @patch.object(broker, 'pool_exists') + @patch.object(broker.Pool, 'remove_cache_tier') + @patch.object(broker, 'log', lambda *args, **kwargs: None) def test_process_requests_remove_cache_tier(self, mock_pool, mock_pool_exists): mock_pool_exists.return_value = True @@ -156,14 +156,14 @@ class TestCephOps(unittest.TestCase): 'op': 'remove-cache-tier', 'hot-pool': 'foo-ssd', }]}) - rc = ceph_broker.process_requests(reqs) + rc = broker.process_requests(reqs) mock_pool_exists.assert_any_call(service='admin', name='foo-ssd') mock_pool.assert_called_with(cache_pool='foo-ssd') self.assertEqual(json.loads(rc), {'exit-code': 0}) - @patch.object(ceph_broker, 'snapshot_pool') - @patch.object(ceph_broker, 'log', lambda *args, **kwargs: None) + @patch.object(broker, 'snapshot_pool') + @patch.object(broker, 'log', lambda *args, **kwargs: None) def test_snapshot_pool(self, mock_snapshot_pool): reqs = json.dumps({'api-version': 1, 'ops': [{ @@ -172,14 +172,14 @@ class TestCephOps(unittest.TestCase): 'snapshot-name': 'foo-snap1', }]}) mock_snapshot_pool.return_value = {'exit-code': 0} - rc = ceph_broker.process_requests(reqs) + rc = broker.process_requests(reqs) mock_snapshot_pool.assert_called_with(service='admin', pool_name='foo', snapshot_name='foo-snap1') self.assertEqual(json.loads(rc), {'exit-code': 0}) - @patch.object(ceph_broker, 'rename_pool') - @patch.object(ceph_broker, 'log', lambda *args, **kwargs: None) + @patch.object(broker, 'rename_pool') + @patch.object(broker, 'log', lambda *args, **kwargs: None) def test_rename_pool(self, mock_rename_pool): reqs = json.dumps({'api-version': 1, 'ops': [{ @@ -188,14 +188,14 @@ class TestCephOps(unittest.TestCase): 'new-name': 'foo2', }]}) mock_rename_pool.return_value = {'exit-code': 0} - rc = ceph_broker.process_requests(reqs) + rc = broker.process_requests(reqs) mock_rename_pool.assert_called_with(service='admin', old_name='foo', new_name='foo2') self.assertEqual(json.loads(rc), {'exit-code': 0}) - @patch.object(ceph_broker, 'remove_pool_snapshot') - @patch.object(ceph_broker, 'log', lambda *args, **kwargs: None) + @patch.object(broker, 'remove_pool_snapshot') + @patch.object(broker, 'log', lambda *args, **kwargs: None) def test_remove_pool_snapshot(self, mock_snapshot_pool): reqs = json.dumps({'api-version': 1, 'ops': [{ @@ -204,14 +204,14 @@ class TestCephOps(unittest.TestCase): 'snapshot-name': 'foo-snap1', }]}) mock_snapshot_pool.return_value = {'exit-code': 0} - rc = ceph_broker.process_requests(reqs) + rc = broker.process_requests(reqs) mock_snapshot_pool.assert_called_with(service='admin', pool_name='foo', snapshot_name='foo-snap1') self.assertEqual(json.loads(rc), {'exit-code': 0}) - @patch.object(ceph_broker, 'pool_set') - @patch.object(ceph_broker, 'log', lambda *args, **kwargs: None) + @patch.object(broker, 'pool_set') + @patch.object(broker, 'log', lambda *args, **kwargs: None) def test_set_pool_value(self, mock_set_pool): reqs = json.dumps({'api-version': 1, 'ops': [{ @@ -221,14 +221,14 @@ class TestCephOps(unittest.TestCase): 'value': 3, }]}) mock_set_pool.return_value = {'exit-code': 0} - rc = ceph_broker.process_requests(reqs) + rc = broker.process_requests(reqs) mock_set_pool.assert_called_with(service='admin', pool_name='foo', key='size', value=3) self.assertEqual(json.loads(rc), {'exit-code': 0}) - @patch.object(ceph_broker, 'log', lambda *args, **kwargs: None) + @patch.object(broker, 'log', lambda *args, **kwargs: None) def test_set_invalid_pool_value(self): reqs = json.dumps({'api-version': 1, 'ops': [{ @@ -237,5 +237,5 @@ class TestCephOps(unittest.TestCase): 'key': 'size', 'value': 'abc', }]}) - rc = ceph_broker.process_requests(reqs) + rc = broker.process_requests(reqs) self.assertEqual(json.loads(rc)['exit-code'], 1)