From 23103d295828e3338859d536779118450cd69683 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Tue, 1 Oct 2019 08:09:47 +0200 Subject: [PATCH] Use charm-internal _upgrade_keyring This change includes a charms.ceph sync. The _upgrade_keyring function was removed from charms.ceph so this charm needs to use the already-existing, charm- internal version. Change-Id: Ia7cf352a2456dc85aca6f61d2e88327cd0c15f7e Closes-Bug: #1845975 --- hooks/ceph_hooks.py | 3 +- lib/ceph/crush_utils.py | 2 +- lib/ceph/utils.py | 209 +++++++++++++++++++++++++--------------- 3 files changed, 132 insertions(+), 82 deletions(-) diff --git a/hooks/ceph_hooks.py b/hooks/ceph_hooks.py index f9903fd6..8de5870e 100755 --- a/hooks/ceph_hooks.py +++ b/hooks/ceph_hooks.py @@ -85,6 +85,7 @@ from utils import ( get_blacklist, get_journal_devices, should_enable_discard, + _upgrade_keyring, ) from charmhelpers.contrib.openstack.alternatives import install_alternative from charmhelpers.contrib.network.ip import ( @@ -124,7 +125,7 @@ CRON_CEPH_CHECK_FILE = '/etc/cron.d/check-osd-services' def check_for_upgrade(): - if not os.path.exists(ceph._upgrade_keyring): + if not os.path.exists(_upgrade_keyring): log("Ceph upgrade keyring not detected, skipping upgrade checks.") return diff --git a/lib/ceph/crush_utils.py b/lib/ceph/crush_utils.py index 8b6876c1..8fe09fa4 100644 --- a/lib/ceph/crush_utils.py +++ b/lib/ceph/crush_utils.py @@ -24,7 +24,7 @@ from charmhelpers.core.hookenv import ( CRUSH_BUCKET = """root {name} {{ id {id} # do not change unnecessarily # weight 0.000 - alg straw + alg straw2 hash 0 # rjenkins1 }} diff --git a/lib/ceph/utils.py b/lib/ceph/utils.py index 31447037..ee555e25 100644 --- a/lib/ceph/utils.py +++ b/lib/ceph/utils.py @@ -40,6 +40,7 @@ from charmhelpers.core.host import ( service_start, service_stop, CompareHostReleases, + write_file, ) from charmhelpers.core.hookenv import ( cached, @@ -928,11 +929,13 @@ def is_osd_disk(dev): def start_osds(devices): # Scan for ceph block devices rescan_osd_devices() - if cmp_pkgrevno('ceph', "0.56.6") >= 0: - # Use ceph-disk activate for directory based OSD's - for dev_or_path in devices: - if os.path.exists(dev_or_path) and os.path.isdir(dev_or_path): - subprocess.check_call(['ceph-disk', 'activate', dev_or_path]) + if (cmp_pkgrevno('ceph', '0.56.6') >= 0 and + cmp_pkgrevno('ceph', '14.2.0') < 0): + # Use ceph-disk activate for directory based OSD's + for dev_or_path in devices: + if os.path.exists(dev_or_path) and os.path.isdir(dev_or_path): + subprocess.check_call( + ['ceph-disk', 'activate', dev_or_path]) def udevadm_settle(): @@ -950,13 +953,12 @@ def rescan_osd_devices(): udevadm_settle() - -_bootstrap_keyring = "/var/lib/ceph/bootstrap-osd/ceph.keyring" -_upgrade_keyring = "/var/lib/ceph/osd/ceph.client.osd-upgrade.keyring" +_client_admin_keyring = '/etc/ceph/ceph.client.admin.keyring' def is_bootstrapped(): - return os.path.exists(_bootstrap_keyring) + return os.path.exists( + '/var/lib/ceph/mon/ceph-{}/done'.format(socket.gethostname())) def wait_for_bootstrap(): @@ -964,36 +966,6 @@ def wait_for_bootstrap(): time.sleep(3) -def import_osd_bootstrap_key(key): - if not os.path.exists(_bootstrap_keyring): - cmd = [ - "sudo", - "-u", - ceph_user(), - 'ceph-authtool', - _bootstrap_keyring, - '--create-keyring', - '--name=client.bootstrap-osd', - '--add-key={}'.format(key) - ] - subprocess.check_call(cmd) - - -def import_osd_upgrade_key(key): - if not os.path.exists(_upgrade_keyring): - cmd = [ - "sudo", - "-u", - ceph_user(), - 'ceph-authtool', - _upgrade_keyring, - '--create-keyring', - '--name=client.osd-upgrade', - '--add-key={}'.format(key) - ] - subprocess.check_call(cmd) - - def generate_monitor_secret(): cmd = [ 'ceph-authtool', @@ -1259,7 +1231,23 @@ def systemd(): return CompareHostReleases(lsb_release()['DISTRIB_CODENAME']) >= 'vivid' +def use_bluestore(): + """Determine whether bluestore should be used for OSD's + + :returns: whether bluestore disk format should be used + :rtype: bool""" + if cmp_pkgrevno('ceph', '12.2.0') < 0: + return False + return config('bluestore') + + def bootstrap_monitor_cluster(secret): + """Bootstrap local ceph mon into the ceph cluster + + :param secret: cephx secret to use for monitor authentication + :type secret: str + :raises: Exception if ceph mon cannot be bootstrapped + """ hostname = socket.gethostname() path = '/var/lib/ceph/mon/ceph-{}'.format(hostname) done = '{}/done'.format(path) @@ -1280,21 +1268,35 @@ def bootstrap_monitor_cluster(secret): perms=0o755) # end changes for Ceph >= 0.61.3 try: - add_keyring_to_ceph(keyring, - secret, - hostname, - path, - done, - init_marker) - + _create_monitor(keyring, + secret, + hostname, + path, + done, + init_marker) + _create_keyrings() except: raise finally: os.unlink(keyring) -@retry_on_exception(3, base_delay=5) -def add_keyring_to_ceph(keyring, secret, hostname, path, done, init_marker): +def _create_monitor(keyring, secret, hostname, path, done, init_marker): + """Create monitor filesystem and enable and start ceph-mon process + + :param keyring: path to temporary keyring on disk + :type keyring: str + :param secret: cephx secret to use for monitor authentication + :type: secret: str + :param hostname: hostname of the local unit + :type hostname: str + :param path: full path to ceph mon directory + :type path: str + :param done: full path to 'done' marker for ceph mon + :type done: str + :param init_marker: full path to 'init' marker for ceph mon + :type init_marker: str + """ subprocess.check_call(['ceph-authtool', keyring, '--create-keyring', '--name=mon.', '--add-key={}'.format(secret), @@ -1310,39 +1312,72 @@ def add_keyring_to_ceph(keyring, secret, hostname, path, done, init_marker): pass if systemd(): - subprocess.check_call(['systemctl', 'enable', 'ceph-mon']) - service_restart('ceph-mon') + if cmp_pkgrevno('ceph', '14.0.0') >= 0: + systemd_unit = 'ceph-mon@{}'.format(socket.gethostname()) + else: + systemd_unit = 'ceph-mon' + subprocess.check_call(['systemctl', 'enable', systemd_unit]) + service_restart(systemd_unit) else: service_restart('ceph-mon-all') - # NOTE(jamespage): Later ceph releases require explicit - # call to ceph-create-keys to setup the - # admin keys for the cluster; this command - # will wait for quorum in the cluster before - # returning. - # NOTE(fnordahl): Explicitly run `ceph-crate-keys` for older - # ceph releases too. This improves bootstrap - # resilience as the charm will wait for - # presence of peer units before attempting - # to bootstrap. Note that charms deploying - # ceph-mon service should disable running of - # `ceph-create-keys` service in init system. - cmd = ['ceph-create-keys', '--id', hostname] - if cmp_pkgrevno('ceph', '12.0.0') >= 0: - # NOTE(fnordahl): The default timeout in ceph-create-keys of 600 - # seconds is not adequate. Increase timeout when - # timeout parameter available. For older releases - # we rely on retry_on_exception decorator. - # LP#1719436 - cmd.extend(['--timeout', '1800']) - subprocess.check_call(cmd) - _client_admin_keyring = '/etc/ceph/ceph.client.admin.keyring' - osstat = os.stat(_client_admin_keyring) - if not osstat.st_size: - # NOTE(fnordahl): Retry will fail as long as this file exists. - # LP#1719436 - os.remove(_client_admin_keyring) - raise Exception + +@retry_on_exception(3, base_delay=5) +def _create_keyrings(): + """Create keyrings for operation of ceph-mon units + + :raises: Exception if keyrings cannot be created + """ + if cmp_pkgrevno('ceph', '14.0.0') >= 0: + # NOTE(jamespage): At Nautilus, keys are created by the + # monitors automatically and just need + # exporting. + output = str(subprocess.check_output( + [ + 'sudo', + '-u', ceph_user(), + 'ceph', + '--name', 'mon.', + '--keyring', + '/var/lib/ceph/mon/ceph-{}/keyring'.format( + socket.gethostname() + ), + 'auth', 'get', 'client.admin', + ]).decode('UTF-8')).strip() + if not output: + # NOTE: key not yet created, raise exception and retry + raise Exception + write_file(_client_admin_keyring, output, + owner=ceph_user(), group=ceph_user(), + perms=0o400) + else: + # NOTE(jamespage): Later ceph releases require explicit + # call to ceph-create-keys to setup the + # admin keys for the cluster; this command + # will wait for quorum in the cluster before + # returning. + # NOTE(fnordahl): Explicitly run `ceph-create-keys` for older + # ceph releases too. This improves bootstrap + # resilience as the charm will wait for + # presence of peer units before attempting + # to bootstrap. Note that charms deploying + # ceph-mon service should disable running of + # `ceph-create-keys` service in init system. + cmd = ['ceph-create-keys', '--id', socket.gethostname()] + if cmp_pkgrevno('ceph', '12.0.0') >= 0: + # NOTE(fnordahl): The default timeout in ceph-create-keys of 600 + # seconds is not adequate. Increase timeout when + # timeout parameter available. For older releases + # we rely on retry_on_exception decorator. + # LP#1719436 + cmd.extend(['--timeout', '1800']) + subprocess.check_call(cmd) + osstat = os.stat(_client_admin_keyring) + if not osstat.st_size: + # NOTE(fnordahl): Retry will fail as long as this file exists. + # LP#1719436 + os.remove(_client_admin_keyring) + raise Exception def update_monfs(): @@ -1427,6 +1462,10 @@ def osdize(dev, osd_format, osd_journal, ignore_errors=False, encrypt=False, ignore_errors, encrypt, bluestore, key_manager) else: + if cmp_pkgrevno('ceph', '14.0.0') >= 0: + log("Directory backed OSDs can not be created on Nautilus", + level=WARNING) + return osdize_dir(dev, encrypt, bluestore) @@ -1555,7 +1594,7 @@ def _ceph_disk(dev, osd_format, osd_journal, encrypt=False, bluestore=False): cmd.append(osd_format) # NOTE(jamespage): enable experimental bluestore support - if cmp_pkgrevno('ceph', '12.2.0') >= 0 and bluestore: + if use_bluestore(): cmd.append('--bluestore') wal = get_devices('bluestore-wal') if wal: @@ -1567,7 +1606,7 @@ def _ceph_disk(dev, osd_format, osd_journal, encrypt=False, bluestore=False): cmd.append('--block.db') least_used_db = find_least_used_utility_device(db) cmd.append(least_used_db) - elif cmp_pkgrevno('ceph', '12.2.0') >= 0 and not bluestore: + elif cmp_pkgrevno('ceph', '12.1.0') >= 0 and not bluestore: cmd.append('--filestore') cmd.append(os.path.realpath(dev)) @@ -2704,6 +2743,14 @@ def dirs_need_ownership_update(service): if (curr_owner == expected_owner) and (curr_group == expected_group): continue + # NOTE(lathiat): when config_changed runs on reboot, the OSD might not + # yet be mounted or started, and the underlying directory the OSD is + # mounted to is expected to be owned by root. So skip the check. This + # may also happen for OSD directories for OSDs that were removed. + if (service == 'osd' and + not os.path.exists(os.path.join(child, 'magic'))): + continue + log('Directory "%s" needs its ownership updated' % child, DEBUG) return True @@ -2716,6 +2763,7 @@ UPGRADE_PATHS = collections.OrderedDict([ ('hammer', 'jewel'), ('jewel', 'luminous'), ('luminous', 'mimic'), + ('mimic', 'nautilus'), ]) # Map UCA codenames to ceph codenames @@ -2731,6 +2779,7 @@ UCA_CODENAME_MAP = { 'queens': 'luminous', 'rocky': 'mimic', 'stein': 'mimic', + 'train': 'nautilus', }