diff --git a/lib/ceph/utils.py b/lib/ceph/utils.py index 1226549..343a759 100644 --- a/lib/ceph/utils.py +++ b/lib/ceph/utils.py @@ -30,6 +30,7 @@ from datetime import datetime from charmhelpers.core import hookenv from charmhelpers.core import templating +from charmhelpers.core.decorators import retry_on_exception from charmhelpers.core.host import ( chownr, cmp_pkgrevno, @@ -51,6 +52,8 @@ from charmhelpers.core.hookenv import ( DEBUG, ERROR, WARNING, + storage_get, + storage_list, ) from charmhelpers.fetch import ( apt_cache, @@ -1285,40 +1288,53 @@ def bootstrap_monitor_cluster(secret): mkdir(path, owner=ceph_user(), group=ceph_user()) # end changes for Ceph >= 0.61.3 try: - subprocess.check_call(['ceph-authtool', keyring, - '--create-keyring', '--name=mon.', - '--add-key={}'.format(secret), - '--cap', 'mon', 'allow *']) + add_keyring_to_ceph(keyring, + secret, + hostname, + path, + done, + init_marker) - subprocess.check_call(['ceph-mon', '--mkfs', - '-i', hostname, - '--keyring', keyring]) - chownr(path, ceph_user(), ceph_user()) - with open(done, 'w'): - pass - with open(init_marker, 'w'): - pass - - if systemd(): - subprocess.check_call(['systemctl', 'enable', 'ceph-mon']) - service_restart('ceph-mon') - else: - service_restart('ceph-mon-all') - - if cmp_pkgrevno('ceph', '12.0.0') >= 0: - # NOTE(jamespage): Later ceph releases require explicit - # call to ceph-create-keys to setup the - # admin keys for the cluster; this command - # will wait for quorum in the cluster before - # returning. - cmd = ['ceph-create-keys', '--id', hostname] - subprocess.check_call(cmd) except: raise finally: os.unlink(keyring) +@retry_on_exception(3, base_delay=5) +def add_keyring_to_ceph(keyring, secret, hostname, path, done, init_marker): + subprocess.check_call(['ceph-authtool', keyring, + '--create-keyring', '--name=mon.', + '--add-key={}'.format(secret), + '--cap', 'mon', 'allow *']) + subprocess.check_call(['ceph-mon', '--mkfs', + '-i', hostname, + '--keyring', keyring]) + chownr(path, ceph_user(), ceph_user()) + with open(done, 'w'): + pass + with open(init_marker, 'w'): + pass + + if systemd(): + subprocess.check_call(['systemctl', 'enable', 'ceph-mon']) + service_restart('ceph-mon') + else: + service_restart('ceph-mon-all') + + if cmp_pkgrevno('ceph', '12.0.0') >= 0: + # NOTE(jamespage): Later ceph releases require explicit + # call to ceph-create-keys to setup the + # admin keys for the cluster; this command + # will wait for quorum in the cluster before + # returning. + cmd = ['ceph-create-keys', '--id', hostname] + subprocess.check_call(cmd) + osstat = os.stat("/etc/ceph/ceph.client.admin.keyring") + if not osstat.st_size: + raise Exception + + def update_monfs(): hostname = socket.gethostname() monfs = '/var/lib/ceph/mon/ceph-{}'.format(hostname) @@ -1353,12 +1369,38 @@ def get_partitions(dev): return [] -def find_least_used_journal(journal_devices): - usages = map(lambda a: (len(get_partitions(a)), a), journal_devices) +def find_least_used_utility_device(utility_devices): + """ + Find a utility device which has the smallest number of partitions + among other devices in the supplied list. + + :utility_devices: A list of devices to be used for filestore journal + or bluestore wal or db. + :return: string device name + """ + + usages = map(lambda a: (len(get_partitions(a)), a), utility_devices) least = min(usages, key=lambda t: t[0]) return least[1] +def get_devices(name): + """ Merge config and juju storage based devices + + :name: THe name of the device type, eg: wal, osd, journal + :returns: Set(device names), which are strings + """ + if config(name): + devices = [l.strip() for l in config(name).split(' ')] + else: + devices = [] + storage_ids = storage_list(name) + devices.extend((storage_get('location', s) for s in storage_ids)) + devices = filter(os.path.exists, devices) + + return set(devices) + + def osdize(dev, osd_format, osd_journal, reformat_osd=False, ignore_errors=False, encrypt=False, bluestore=False): if dev.startswith('/dev'): @@ -1405,13 +1447,23 @@ def osdize_dev(dev, osd_format, osd_journal, reformat_osd=False, # NOTE(jamespage): enable experimental bluestore support if cmp_pkgrevno('ceph', '10.2.0') >= 0 and bluestore: cmd.append('--bluestore') + wal = get_devices('bluestore-wal') + if wal: + cmd.append('--block.wal') + least_used_wal = find_least_used_utility_device(wal) + cmd.append(least_used_wal) + db = get_devices('bluestore-db') + if db: + cmd.append('--block.db') + least_used_db = find_least_used_utility_device(db) + cmd.append(least_used_db) elif cmp_pkgrevno('ceph', '12.1.0') >= 0 and not bluestore: cmd.append('--filestore') cmd.append(dev) if osd_journal: - least_used = find_least_used_journal(osd_journal) + least_used = find_least_used_utility_device(osd_journal) cmd.append(least_used) else: # Just provide the device - no other options