Add support for Nautilus release
Resync charms.ceph to pickup changes to support new approach to admin and bootstrap key generation with in the ceph-mon cluster. Update get_mon_hosts to avoid specifying the port number for the MON daemons; we use the default so this is not required, and at Nautilus the MON daemons run both v1 and v2 messenger ports. Specifying the port in the ceph.conf file disables the v2 messenger port which is not the desired behaviour on upgrade or new installation. Drop start hook; this has been present in the charm since its inception as 'ceph' but is really not needed - re-deploying the charm to units which previous hosted ceph-mon is dangerous and the ceph-* daemons should still be running anyway. Depends-On: I2dfab7404b72e314625ea554ee64595c5e26f3c6 Change-Id: I340dbf427551e6f9f8cd4ca89128627e497d6097
This commit is contained in:
parent
63b38bf5ce
commit
872c670ed6
|
@ -18,7 +18,6 @@ import ast
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import subprocess
|
import subprocess
|
||||||
import socket
|
|
||||||
import sys
|
import sys
|
||||||
import uuid
|
import uuid
|
||||||
|
|
||||||
|
@ -51,7 +50,6 @@ from charmhelpers.core.hookenv import (
|
||||||
application_version_set)
|
application_version_set)
|
||||||
from charmhelpers.core.host import (
|
from charmhelpers.core.host import (
|
||||||
service_pause,
|
service_pause,
|
||||||
service_restart,
|
|
||||||
mkdir,
|
mkdir,
|
||||||
write_file,
|
write_file,
|
||||||
rsync,
|
rsync,
|
||||||
|
@ -298,7 +296,7 @@ def config_changed():
|
||||||
def get_mon_hosts():
|
def get_mon_hosts():
|
||||||
hosts = []
|
hosts = []
|
||||||
addr = get_public_addr()
|
addr = get_public_addr()
|
||||||
hosts.append('{}:6789'.format(format_ipv6_addr(addr) or addr))
|
hosts.append(format_ipv6_addr(addr) or addr)
|
||||||
|
|
||||||
rel_ids = relation_ids('mon')
|
rel_ids = relation_ids('mon')
|
||||||
if config('no-bootstrap'):
|
if config('no-bootstrap'):
|
||||||
|
@ -308,8 +306,7 @@ def get_mon_hosts():
|
||||||
for unit in related_units(relid):
|
for unit in related_units(relid):
|
||||||
addr = relation_get('ceph-public-address', unit, relid)
|
addr = relation_get('ceph-public-address', unit, relid)
|
||||||
if addr is not None:
|
if addr is not None:
|
||||||
hosts.append('{}:6789'.format(
|
hosts.append(format_ipv6_addr(addr) or addr)
|
||||||
format_ipv6_addr(addr) or addr))
|
|
||||||
|
|
||||||
return sorted(hosts)
|
return sorted(hosts)
|
||||||
|
|
||||||
|
@ -818,18 +815,6 @@ def upgrade_charm():
|
||||||
notify_rbd_mirrors()
|
notify_rbd_mirrors()
|
||||||
|
|
||||||
|
|
||||||
@hooks.hook('start')
|
|
||||||
def start():
|
|
||||||
# In case we're being redeployed to the same machines, try
|
|
||||||
# to make sure everything is running as soon as possible.
|
|
||||||
if ceph.systemd():
|
|
||||||
service_restart('ceph-mon')
|
|
||||||
else:
|
|
||||||
service_restart('ceph-mon-all')
|
|
||||||
if cmp_pkgrevno('ceph', '12.0.0') >= 0:
|
|
||||||
service_restart('ceph-mgr@{}'.format(socket.gethostname()))
|
|
||||||
|
|
||||||
|
|
||||||
@hooks.hook('nrpe-external-master-relation-joined')
|
@hooks.hook('nrpe-external-master-relation-joined')
|
||||||
@hooks.hook('nrpe-external-master-relation-changed')
|
@hooks.hook('nrpe-external-master-relation-changed')
|
||||||
def update_nrpe_config():
|
def update_nrpe_config():
|
||||||
|
|
|
@ -40,6 +40,7 @@ from charmhelpers.core.host import (
|
||||||
service_start,
|
service_start,
|
||||||
service_stop,
|
service_stop,
|
||||||
CompareHostReleases,
|
CompareHostReleases,
|
||||||
|
write_file,
|
||||||
)
|
)
|
||||||
from charmhelpers.core.hookenv import (
|
from charmhelpers.core.hookenv import (
|
||||||
cached,
|
cached,
|
||||||
|
@ -82,7 +83,7 @@ QUORUM = [LEADER, PEON]
|
||||||
|
|
||||||
PACKAGES = ['ceph', 'gdisk', 'btrfs-tools',
|
PACKAGES = ['ceph', 'gdisk', 'btrfs-tools',
|
||||||
'radosgw', 'xfsprogs',
|
'radosgw', 'xfsprogs',
|
||||||
'lvm2', 'parted']
|
'lvm2', 'parted', 'smartmontools']
|
||||||
|
|
||||||
CEPH_KEY_MANAGER = 'ceph'
|
CEPH_KEY_MANAGER = 'ceph'
|
||||||
VAULT_KEY_MANAGER = 'vault'
|
VAULT_KEY_MANAGER = 'vault'
|
||||||
|
@ -950,13 +951,13 @@ def rescan_osd_devices():
|
||||||
|
|
||||||
udevadm_settle()
|
udevadm_settle()
|
||||||
|
|
||||||
|
_client_admin_keyring = '/etc/ceph/ceph.client.admin.keyring'
|
||||||
_bootstrap_keyring = "/var/lib/ceph/bootstrap-osd/ceph.keyring"
|
_bootstrap_keyring = "/var/lib/ceph/bootstrap-osd/ceph.keyring"
|
||||||
_upgrade_keyring = "/var/lib/ceph/osd/ceph.client.osd-upgrade.keyring"
|
_upgrade_keyring = "/var/lib/ceph/osd/ceph.client.osd-upgrade.keyring"
|
||||||
|
|
||||||
|
|
||||||
def is_bootstrapped():
|
def is_bootstrapped():
|
||||||
return os.path.exists(_bootstrap_keyring)
|
return os.path.exists(_client_admin_keyring)
|
||||||
|
|
||||||
|
|
||||||
def wait_for_bootstrap():
|
def wait_for_bootstrap():
|
||||||
|
@ -1259,7 +1260,23 @@ def systemd():
|
||||||
return CompareHostReleases(lsb_release()['DISTRIB_CODENAME']) >= 'vivid'
|
return CompareHostReleases(lsb_release()['DISTRIB_CODENAME']) >= 'vivid'
|
||||||
|
|
||||||
|
|
||||||
|
def use_bluestore():
|
||||||
|
"""Determine whether bluestore should be used for OSD's
|
||||||
|
|
||||||
|
:returns: whether bluestore disk format should be used
|
||||||
|
:rtype: bool"""
|
||||||
|
if cmp_pkgrevno('ceph', '12.2.0') < 0:
|
||||||
|
return False
|
||||||
|
return config('bluestore')
|
||||||
|
|
||||||
|
|
||||||
def bootstrap_monitor_cluster(secret):
|
def bootstrap_monitor_cluster(secret):
|
||||||
|
"""Bootstrap local ceph mon into the ceph cluster
|
||||||
|
|
||||||
|
:param secret: cephx secret to use for monitor authentication
|
||||||
|
:type secret: str
|
||||||
|
:raises: Exception if ceph mon cannot be bootstrapped
|
||||||
|
"""
|
||||||
hostname = socket.gethostname()
|
hostname = socket.gethostname()
|
||||||
path = '/var/lib/ceph/mon/ceph-{}'.format(hostname)
|
path = '/var/lib/ceph/mon/ceph-{}'.format(hostname)
|
||||||
done = '{}/done'.format(path)
|
done = '{}/done'.format(path)
|
||||||
|
@ -1280,21 +1297,35 @@ def bootstrap_monitor_cluster(secret):
|
||||||
perms=0o755)
|
perms=0o755)
|
||||||
# end changes for Ceph >= 0.61.3
|
# end changes for Ceph >= 0.61.3
|
||||||
try:
|
try:
|
||||||
add_keyring_to_ceph(keyring,
|
_create_monitor(keyring,
|
||||||
secret,
|
secret,
|
||||||
hostname,
|
hostname,
|
||||||
path,
|
path,
|
||||||
done,
|
done,
|
||||||
init_marker)
|
init_marker)
|
||||||
|
_create_keyrings()
|
||||||
except:
|
except:
|
||||||
raise
|
raise
|
||||||
finally:
|
finally:
|
||||||
os.unlink(keyring)
|
os.unlink(keyring)
|
||||||
|
|
||||||
|
|
||||||
@retry_on_exception(3, base_delay=5)
|
def _create_monitor(keyring, secret, hostname, path, done, init_marker):
|
||||||
def add_keyring_to_ceph(keyring, secret, hostname, path, done, init_marker):
|
"""Create monitor filesystem and enable and start ceph-mon process
|
||||||
|
|
||||||
|
:param keyring: path to temporary keyring on disk
|
||||||
|
:type keyring: str
|
||||||
|
:param secret: cephx secret to use for monitor authentication
|
||||||
|
:type: secret: str
|
||||||
|
:param hostname: hostname of the local unit
|
||||||
|
:type hostname: str
|
||||||
|
:param path: full path to ceph mon directory
|
||||||
|
:type path: str
|
||||||
|
:param done: full path to 'done' marker for ceph mon
|
||||||
|
:type done: str
|
||||||
|
:param init_marker: full path to 'init' marker for ceph mon
|
||||||
|
:type init_marker: str
|
||||||
|
"""
|
||||||
subprocess.check_call(['ceph-authtool', keyring,
|
subprocess.check_call(['ceph-authtool', keyring,
|
||||||
'--create-keyring', '--name=mon.',
|
'--create-keyring', '--name=mon.',
|
||||||
'--add-key={}'.format(secret),
|
'--add-key={}'.format(secret),
|
||||||
|
@ -1310,39 +1341,72 @@ def add_keyring_to_ceph(keyring, secret, hostname, path, done, init_marker):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
if systemd():
|
if systemd():
|
||||||
subprocess.check_call(['systemctl', 'enable', 'ceph-mon'])
|
if cmp_pkgrevno('ceph', '14.0.0') >= 0:
|
||||||
service_restart('ceph-mon')
|
systemd_unit = 'ceph-mon@{}'.format(socket.gethostname())
|
||||||
|
else:
|
||||||
|
systemd_unit = 'ceph-mon'
|
||||||
|
subprocess.check_call(['systemctl', 'enable', systemd_unit])
|
||||||
|
service_restart(systemd_unit)
|
||||||
else:
|
else:
|
||||||
service_restart('ceph-mon-all')
|
service_restart('ceph-mon-all')
|
||||||
|
|
||||||
# NOTE(jamespage): Later ceph releases require explicit
|
|
||||||
# call to ceph-create-keys to setup the
|
@retry_on_exception(3, base_delay=5)
|
||||||
# admin keys for the cluster; this command
|
def _create_keyrings():
|
||||||
# will wait for quorum in the cluster before
|
"""Create keyrings for operation of ceph-mon units
|
||||||
# returning.
|
|
||||||
# NOTE(fnordahl): Explicitly run `ceph-crate-keys` for older
|
:raises: Exception if keyrings cannot be created
|
||||||
# ceph releases too. This improves bootstrap
|
"""
|
||||||
# resilience as the charm will wait for
|
if cmp_pkgrevno('ceph', '14.0.0') >= 0:
|
||||||
# presence of peer units before attempting
|
# NOTE(jamespage): At Nautilus, keys are created by the
|
||||||
# to bootstrap. Note that charms deploying
|
# monitors automatically and just need
|
||||||
# ceph-mon service should disable running of
|
# exporting.
|
||||||
# `ceph-create-keys` service in init system.
|
output = str(subprocess.check_output(
|
||||||
cmd = ['ceph-create-keys', '--id', hostname]
|
[
|
||||||
if cmp_pkgrevno('ceph', '12.0.0') >= 0:
|
'sudo',
|
||||||
# NOTE(fnordahl): The default timeout in ceph-create-keys of 600
|
'-u', ceph_user(),
|
||||||
# seconds is not adequate. Increase timeout when
|
'ceph',
|
||||||
# timeout parameter available. For older releases
|
'--name', 'mon.',
|
||||||
# we rely on retry_on_exception decorator.
|
'--keyring',
|
||||||
# LP#1719436
|
'/var/lib/ceph/mon/ceph-{}/keyring'.format(
|
||||||
cmd.extend(['--timeout', '1800'])
|
socket.gethostname()
|
||||||
subprocess.check_call(cmd)
|
),
|
||||||
_client_admin_keyring = '/etc/ceph/ceph.client.admin.keyring'
|
'auth', 'get', 'client.admin',
|
||||||
osstat = os.stat(_client_admin_keyring)
|
]).decode('UTF-8')).strip()
|
||||||
if not osstat.st_size:
|
if not output:
|
||||||
# NOTE(fnordahl): Retry will fail as long as this file exists.
|
# NOTE: key not yet created, raise exception and retry
|
||||||
# LP#1719436
|
raise Exception
|
||||||
os.remove(_client_admin_keyring)
|
write_file(_client_admin_keyring, output,
|
||||||
raise Exception
|
owner=ceph_user(), group=ceph_user(),
|
||||||
|
perms=0o400)
|
||||||
|
else:
|
||||||
|
# NOTE(jamespage): Later ceph releases require explicit
|
||||||
|
# call to ceph-create-keys to setup the
|
||||||
|
# admin keys for the cluster; this command
|
||||||
|
# will wait for quorum in the cluster before
|
||||||
|
# returning.
|
||||||
|
# NOTE(fnordahl): Explicitly run `ceph-create-keys` for older
|
||||||
|
# ceph releases too. This improves bootstrap
|
||||||
|
# resilience as the charm will wait for
|
||||||
|
# presence of peer units before attempting
|
||||||
|
# to bootstrap. Note that charms deploying
|
||||||
|
# ceph-mon service should disable running of
|
||||||
|
# `ceph-create-keys` service in init system.
|
||||||
|
cmd = ['ceph-create-keys', '--id', socket.gethostname()]
|
||||||
|
if cmp_pkgrevno('ceph', '12.0.0') >= 0:
|
||||||
|
# NOTE(fnordahl): The default timeout in ceph-create-keys of 600
|
||||||
|
# seconds is not adequate. Increase timeout when
|
||||||
|
# timeout parameter available. For older releases
|
||||||
|
# we rely on retry_on_exception decorator.
|
||||||
|
# LP#1719436
|
||||||
|
cmd.extend(['--timeout', '1800'])
|
||||||
|
subprocess.check_call(cmd)
|
||||||
|
osstat = os.stat(_client_admin_keyring)
|
||||||
|
if not osstat.st_size:
|
||||||
|
# NOTE(fnordahl): Retry will fail as long as this file exists.
|
||||||
|
# LP#1719436
|
||||||
|
os.remove(_client_admin_keyring)
|
||||||
|
raise Exception
|
||||||
|
|
||||||
|
|
||||||
def update_monfs():
|
def update_monfs():
|
||||||
|
@ -1555,7 +1619,7 @@ def _ceph_disk(dev, osd_format, osd_journal, encrypt=False, bluestore=False):
|
||||||
cmd.append(osd_format)
|
cmd.append(osd_format)
|
||||||
|
|
||||||
# NOTE(jamespage): enable experimental bluestore support
|
# NOTE(jamespage): enable experimental bluestore support
|
||||||
if cmp_pkgrevno('ceph', '10.2.0') >= 0 and bluestore:
|
if use_bluestore():
|
||||||
cmd.append('--bluestore')
|
cmd.append('--bluestore')
|
||||||
wal = get_devices('bluestore-wal')
|
wal = get_devices('bluestore-wal')
|
||||||
if wal:
|
if wal:
|
||||||
|
@ -1692,7 +1756,10 @@ def is_active_bluestore_device(dev):
|
||||||
return False
|
return False
|
||||||
|
|
||||||
vg_name = lvm.list_lvm_volume_group(dev)
|
vg_name = lvm.list_lvm_volume_group(dev)
|
||||||
lv_name = lvm.list_logical_volumes('vg_name={}'.format(vg_name))[0]
|
try:
|
||||||
|
lv_name = lvm.list_logical_volumes('vg_name={}'.format(vg_name))[0]
|
||||||
|
except IndexError:
|
||||||
|
return False
|
||||||
|
|
||||||
block_symlinks = glob.glob('/var/lib/ceph/osd/ceph-*/block')
|
block_symlinks = glob.glob('/var/lib/ceph/osd/ceph-*/block')
|
||||||
for block_candidate in block_symlinks:
|
for block_candidate in block_symlinks:
|
||||||
|
@ -2700,6 +2767,14 @@ def dirs_need_ownership_update(service):
|
||||||
if (curr_owner == expected_owner) and (curr_group == expected_group):
|
if (curr_owner == expected_owner) and (curr_group == expected_group):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
# NOTE(lathiat): when config_changed runs on reboot, the OSD might not
|
||||||
|
# yet be mounted or started, and the underlying directory the OSD is
|
||||||
|
# mounted to is expected to be owned by root. So skip the check. This
|
||||||
|
# may also happen for OSD directories for OSDs that were removed.
|
||||||
|
if (service == 'osd' and
|
||||||
|
not os.path.exists(os.path.join(child, 'magic'))):
|
||||||
|
continue
|
||||||
|
|
||||||
log('Directory "%s" needs its ownership updated' % child, DEBUG)
|
log('Directory "%s" needs its ownership updated' % child, DEBUG)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
@ -2712,6 +2787,8 @@ UPGRADE_PATHS = collections.OrderedDict([
|
||||||
('hammer', 'jewel'),
|
('hammer', 'jewel'),
|
||||||
('jewel', 'luminous'),
|
('jewel', 'luminous'),
|
||||||
('luminous', 'mimic'),
|
('luminous', 'mimic'),
|
||||||
|
('luminous', 'nautilus'),
|
||||||
|
('mimic', 'nautilus'),
|
||||||
])
|
])
|
||||||
|
|
||||||
# Map UCA codenames to ceph codenames
|
# Map UCA codenames to ceph codenames
|
||||||
|
@ -2727,6 +2804,7 @@ UCA_CODENAME_MAP = {
|
||||||
'queens': 'luminous',
|
'queens': 'luminous',
|
||||||
'rocky': 'mimic',
|
'rocky': 'mimic',
|
||||||
'stein': 'mimic',
|
'stein': 'mimic',
|
||||||
|
'train': 'nautilus',
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -619,8 +619,8 @@ class BootstrapSourceTestCase(test_utils.CharmTestCase):
|
||||||
self.relation_get.side_effect = rel_get_side_effect
|
self.relation_get.side_effect = rel_get_side_effect
|
||||||
hosts = ceph_hooks.get_mon_hosts()
|
hosts = ceph_hooks.get_mon_hosts()
|
||||||
self.assertEqual(hosts, [
|
self.assertEqual(hosts, [
|
||||||
'172.16.0.2:6789', '172.16.0.3:6789', '172.16.0.4:6789',
|
'172.16.0.2', '172.16.0.3', '172.16.0.4',
|
||||||
'172.16.10.2:6789', '172.16.10.3:6789', '172.16.10.4:6789',
|
'172.16.10.2', '172.16.10.3', '172.16.10.4',
|
||||||
])
|
])
|
||||||
|
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue